• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 19707535375

26 Nov 2025 02:42PM UTC coverage: 69.22% (-0.3%) from 69.481%
19707535375

Pull #1219

github

web-flow
Merge 26ecb95d8 into ea0653a4e
Pull Request #1219: !!! [TASK] Remove PHP 8.1 support

13 of 15 new or added lines in 6 files covered. (86.67%)

7 existing lines in 2 files now uncovered.

1934 of 2794 relevant lines covered (69.22%)

3.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleCrawlerLogController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\ResultHandler;
24
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
25
use AOE\Crawler\Controller\CrawlerController;
26
use AOE\Crawler\Converter\JsonCompatibilityConverter;
27
use AOE\Crawler\Domain\Repository\QueueRepository;
28
use AOE\Crawler\Service\BackendModuleLogService;
29
use AOE\Crawler\Service\BackendModuleScriptUrlService;
30
use AOE\Crawler\Utility\MessageUtility;
31
use AOE\Crawler\Value\QueueFilter;
32
use AOE\Crawler\Writer\FileWriter\CsvWriter\CsvWriterInterface;
33
use Psr\Http\Message\ResponseInterface;
34
use Psr\Http\Message\ServerRequestInterface;
35
use Psr\Http\Message\UriInterface;
36
use Symfony\Contracts\Service\Attribute\Required;
37
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
38
use TYPO3\CMS\Backend\Template\ModuleTemplate;
39
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
40
use TYPO3\CMS\Backend\Utility\BackendUtility;
41
use TYPO3\CMS\Core\Database\ConnectionPool;
42
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
43
use TYPO3\CMS\Core\Imaging\IconFactory;
44
use TYPO3\CMS\Core\Imaging\IconSize;
45
use TYPO3\CMS\Core\Utility\DebugUtility;
46
use TYPO3\CMS\Core\Utility\GeneralUtility;
47

48
/**
49
 * @internal since v12.0.0
50
 */
51
final class BackendModuleCrawlerLogController extends AbstractBackendModuleController implements BackendModuleControllerInterface
52
{
53
    public const BACKEND_MODULE = 'web_site_crawler_log';
54

55
    private QueryBuilder $queryBuilder;
56
    private bool $CSVExport = false;
57
    private readonly array $backendModuleMenu;
58
    private int $setId;
59
    private string $quiPath;
60
    private string $logDisplay;
61
    private int $itemsPerPage;
62
    private string $showResultLog;
63
    private string $showFeVars;
64
    private int $showSetId;
65
    private string $logDepth;
66
    /**
67
     * @var mixed|string|null
68
     */
69
    private mixed $queueId;
70

71
    public function __construct(
72
        private readonly QueueRepository $queueRepository,
73
        private readonly CsvWriterInterface $csvWriter,
74
        private readonly JsonCompatibilityConverter $jsonCompatibilityConverter,
75
        private readonly IconFactory $iconFactory,
76
        private readonly CrawlerController $crawlerController,
77
        private readonly BackendModuleLogService $backendModuleLogService,
78
        private readonly BackendModuleScriptUrlService $backendModuleScriptUrlService,
79
    ) {
80
        $this->backendModuleMenu = $this->getModuleMenu();
×
81
    }
82

83
    #[Required]
84
    public function setQueryBuilder(): void
85
    {
86
        $this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(
×
87
            QueueRepository::TABLE_NAME
×
88
        );
×
89
    }
90

91
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
92
    {
93
        $this->setPropertiesBasedOnPostVars($request);
×
94
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
95
        $this->moduleTemplate = $this->moduleTemplate->makeDocHeaderModuleMenu([
×
96
            'id' => $this->pageUid,
×
97
        ]);
×
98

99
        if (!$this->pageUid) {
×
100
            $this->isErrorDetected = true;
×
101
            $this->moduleTemplate->assign('noPageSelected', true);
×
102
            return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
103
        }
104
        $this->moduleTemplate = $this->assignValues($request);
×
105
        return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
106
    }
107

108
    private function getQueueEntry(mixed $queueId): array
109
    {
110
        $q_entry = $this->queryBuilder
×
111
            ->from(QueueRepository::TABLE_NAME)
×
112
            ->select('*')->where(
×
113
                $this->queryBuilder->expr()->eq('qid', $this->queryBuilder->createNamedParameter($queueId))
×
114
            )->executeQuery()
×
115
            ->fetchAssociative();
×
116

117
        // Explode values
118
        $q_entry['parameters'] = $this->jsonCompatibilityConverter->convert($q_entry['parameters']);
×
119
        $q_entry['result_data'] = $this->jsonCompatibilityConverter->convert($q_entry['result_data']);
×
120
        $resStatus = ResultHandler::getResStatus($q_entry['result_data']);
×
121
        if (is_array($q_entry['result_data'])) {
×
122
            $q_entry['result_data']['content'] = $this->jsonCompatibilityConverter->convert(
×
123
                $q_entry['result_data']['content']
×
124
            );
×
125
            if (!$this->showResultLog) {
×
126
                if (is_array($q_entry['result_data']['content'])) {
×
127
                    unset($q_entry['result_data']['content']['log']);
×
128
                }
129
            }
130
        }
131
        return [$q_entry, $resStatus];
×
132
    }
133

134
    /**
135
     * @throws RouteNotFoundException
136
     */
137
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
138
    {
139
        // Look for set ID sent - if it is, we will display contents of that set:
140
        $this->showSetId = RequestHelper::getIntFromRequest($request, 'setID');
×
141
        $this->CSVExport = RequestHelper::getBoolFromRequest($request, '_csv');
×
142
        $logEntriesPerPage = [];
×
143
        $csvData = [];
×
144

145
        $quidRead = RequestHelper::getIntFromRequest($request, 'qid_read');
×
146
        if ($quidRead) {
×
147
            $this->crawlerController->readUrl($quidRead, true);
×
148
        }
149

150
        if ($this->queueId) {
×
151
            // Get entry record:
152
            [$q_entry, $resStatus] = $this->getQueueEntry($this->queueId);
×
153
            $this->moduleTemplate->assignMultiple([
×
154
                'queueStatus' => $resStatus,
×
155
                'queueDetails' => DebugUtility::viewArray($q_entry),
×
156
            ]);
×
157
        } else {
158
            // Show list
159
            // Drawing tree:
160
            $tree = GeneralUtility::makeInstance(PageTreeView::class);
×
161
            $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
×
162
            $tree->init('AND ' . $perms_clause);
×
163

164
            // Set root row:
165
            $pageinfo = BackendUtility::readPageAccess($this->pageUid, $perms_clause);
×
166

167
            if (is_array($pageinfo)) {
×
NEW
168
                $HTML = $this->iconFactory->getIconForRecord('pages', $pageinfo, IconSize::SMALL)->render();
×
169
                $tree->tree[] = [
×
170
                    'row' => $pageinfo,
×
171
                    'HTML' => $HTML,
×
172
                ];
×
173
            }
174

175
            // Get branch beneath:
176
            if ($this->logDepth) {
×
177
                $tree->getTree($this->pageUid, (int) $this->logDepth);
×
178
            }
179

180
            // If Flush button is pressed, flush tables instead of selecting entries:
181
            if ($request->getParsedBody()['_flush'] ?? false) {
×
182
                $doFlush = true;
×
183
            } elseif ($request->getParsedBody()['_flush_all'] ?? false) {
×
184
                $doFlush = true;
×
185
                $this->logDisplay = 'all';
×
186
            } else {
187
                $doFlush = false;
×
188
            }
189

190
            $queueFilter = new QueueFilter($this->logDisplay);
×
191

192
            if ($doFlush) {
×
193
                $this->queueRepository->flushQueue($queueFilter);
×
194
            }
195

196
            // Traverse page tree:
197
            $count = 0;
×
198

199
            foreach ($tree->tree as $data) {
×
200
                $logEntriesOfPage = $this->queueRepository->getQueueEntriesForPageId(
×
201
                    (int) $data['row']['uid'],
×
202
                    $this->itemsPerPage,
×
203
                    $queueFilter
×
204
                );
×
205

206
                [$logEntriesPerPage[], $row] = $this->backendModuleLogService->addRows(
×
207
                    $logEntriesOfPage,
×
208
                    RequestHelper::getIntFromRequest($request, 'setID'),
×
209
                    $data['HTML'] . BackendUtility::getRecordTitle('pages', $data['row'], true),
×
210
                    $this->showResultLog,
×
211
                    $this->showFeVars,
×
212
                    $this->CSVExport
×
213
                );
×
214
                $csvData[] = $row;
×
215

216
                if (++$count === 1000) {
×
217
                    break;
×
218
                }
219
            }
220

221
            $this->moduleTemplate->assign('logEntriesPerPage', $logEntriesPerPage);
×
222
        }
223

224
        if ($this->CSVExport) {
×
225
            $this->outputCsvFile($csvData);
×
226
        }
227

228
        $queryParams = [
×
229
            'setID' => $this->setId,
×
230
            'displayLog' => $this->logDisplay,
×
231
            'itemsPerPage' => $this->itemsPerPage,
×
232
            'ShowFeVars' => $this->showFeVars,
×
233
            'ShowResultLog' => $this->showResultLog,
×
234
            'logDepth' => $this->logDepth,
×
235
        ];
×
236

237
        return $this->moduleTemplate->assignMultiple([
×
238
            'backendModuleUrl' => $this->getBackendModuleUrl(),
×
239
            'actionUrl' => '',
×
240
            'queueId' => $this->queueId,
×
241
            'setId' => $this->showSetId,
×
242
            'noPageSelected' => false,
×
243
            'logEntriesPerPage' => $logEntriesPerPage,
×
244
            'showResultLog' => $this->showResultLog,
×
245
            'showFeVars' => $this->showFeVars,
×
246
            'displayActions' => 1,
×
247
            'displayLogFilterConfig' => [
×
248
                'name' => 'displayLog',
×
249
                'currentValue' => $this->logDisplay,
×
250
                'menuItems' => $this->backendModuleMenu['displayLog'],
×
251
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
252
                    $request,
×
253
                    'displayLog',
×
254
                    $this->pageUid,
×
255
                    $queryParams
×
256
                ),
×
257
            ],
×
258
            'itemPerPageConfig' => [
×
259
                'name' => 'itemsPerPage',
×
260
                'currentValue' => $this->itemsPerPage,
×
261
                'menuItems' => $this->backendModuleMenu['itemsPerPage'],
×
262
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
263
                    $request,
×
264
                    'itemsPerPage',
×
265
                    $this->pageUid,
×
266
                    $queryParams
×
267
                ),
×
268
            ],
×
269
            'showResultLogConfig' => [
×
270
                'name' => 'ShowResultLog',
×
271
                'currentValue' => $this->showResultLog,
×
272
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
273
                    $request,
×
274
                    'ShowResultLog',
×
275
                    $this->pageUid,
×
276
                    $queryParams,
×
277
                    $this->quiPath
×
278
                ),
×
279
            ],
×
280
            'showFeVarsConfig' => [
×
281
                'name' => 'ShowFeVars',
×
282
                'currentValue' => $this->showFeVars,
×
283
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
284
                    $request,
×
285
                    'ShowFeVars',
×
286
                    $this->pageUid,
×
287
                    $queryParams,
×
288
                    $this->quiPath
×
289
                ),
×
290
            ],
×
291
            'depthDropDownConfig' => [
×
292
                'name' => 'logDepth',
×
293
                'currentValue' => $this->logDepth,
×
294
                'menuItems' => $this->backendModuleMenu['logDepth'],
×
295
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
296
                    $request,
×
297
                    'logDepth',
×
298
                    $this->pageUid,
×
299
                    $queryParams
×
300
                ),
×
301
            ],
×
302
        ]);
×
303
    }
304

305
    private function setPropertiesBasedOnPostVars(ServerRequestInterface $request): void
306
    {
307
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
308
        $this->setId = RequestHelper::getIntFromRequest($request, 'setIDÍ„');
×
309
        $quidDetails = $request->getParsedBody()['qid_details'] ?? $request->getQueryParams()['qid_details'] ?? null;
×
310
        $this->quiPath = $quidDetails ? '&qid_details=' . (int) $quidDetails : '';
×
311
        $this->queueId = $quidDetails ?? null;
×
312
        $this->logDisplay = RequestHelper::getStringFromRequest($request, 'displayLog', 'all');
×
313
        $this->itemsPerPage = RequestHelper::getIntFromRequest($request, 'itemsPerPage', 10);
×
314
        $this->showResultLog = RequestHelper::getStringFromRequest($request, 'ShowResultLog', '0');
×
315
        $this->showFeVars = RequestHelper::getStringFromRequest($request, 'ShowFeVars', '0');
×
316
        $this->logDepth = RequestHelper::getStringFromRequest($request, 'logDepth', '0');
×
317
    }
318

319
    /**
320
     * Outputs the CSV file and sets the correct headers
321
     */
322
    private function outputCsvFile(array $csvData): void
323
    {
324
        if (!count($csvData)) {
×
325
            MessageUtility::addWarningMessage(
×
326
                $this->getLanguageService()->sL(
×
327
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:message.canNotExportEmptyQueueToCsvText'
×
328
                )
×
329
            );
×
330
            return;
×
331
        }
332

333
        $csvString = $this->csvWriter->arrayToCsv($csvData);
×
334

335
        header('Content-Type: application/octet-stream');
×
336
        header('Content-Disposition: attachment; filename=CrawlerLog.csv');
×
337
        echo $csvString;
×
338

339
        exit;
×
340
    }
341

342
    private function getBackendModuleUrl(): UriInterface
343
    {
344
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
345
    }
346
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc