• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43495013365

13 Nov 2026 07:44PM UTC coverage: 66.983% (-0.2%) from 67.139%
43495013365

push

github

web-flow
[BUGFIX] Export every log entry in the crawler log CSV, reduce BackendModuleLogService complexity (#1326)

* [TASK] Add test showing the CSV log export drops entries

BackendModuleLogService::addRows() overwrites the CSV row on every log
entry, so only the last entry of each page ends up in the export. This
test currently fails and documents the bug.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* [BUGFIX] Export every log entry in the crawler log CSV

addRows() now returns one CSV row per log entry instead of overwriting
the row on every iteration, and the log controller merges those rows
into a flat list. Pages without log entries no longer add an empty
row to the export.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* [TASK] Reduce cognitive complexity of BackendModuleLogService

Split addRows() into buildRow(), buildRowData(), formatDateTime() and
getModuleUrl(), and return early when a page has no log entries. The
rendered rows and CSV rows are unchanged.

This removes the two cognitive complexity entries for the class from
the PHPStan baseline. The new private methods are added to the Psalm
baseline, like the existing helpers, because Psalm cannot see the
dependency-injected caller of addRows().

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

59 of 62 new or added lines in 2 files covered. (95.16%)

1272 of 1899 relevant lines covered (66.98%)

5.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleCrawlerLogController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\ResultHandler;
24
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
25
use AOE\Crawler\Domain\Repository\QueueRepository;
26
use AOE\Crawler\Queue\CrawlPriority;
27
use AOE\Crawler\Queue\CrawlUrlMessageDispatcher;
28
use AOE\Crawler\Queue\Message\CrawlUrlMessage;
29
use AOE\Crawler\Service\BackendModuleLogService;
30
use AOE\Crawler\Service\BackendModuleScriptUrlService;
31
use AOE\Crawler\Utility\MessageUtility;
32
use AOE\Crawler\Value\QueueFilter;
33
use AOE\Crawler\Writer\FileWriter\CsvWriter\CsvWriterInterface;
34
use Psr\Http\Message\ResponseInterface;
35
use Psr\Http\Message\ServerRequestInterface;
36
use Psr\Http\Message\UriInterface;
37
use Symfony\Contracts\Service\Attribute\Required;
38
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
39
use TYPO3\CMS\Backend\Template\ModuleTemplate;
40
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
41
use TYPO3\CMS\Backend\Utility\BackendUtility;
42
use TYPO3\CMS\Core\Database\ConnectionPool;
43
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
44
use TYPO3\CMS\Core\Imaging\IconFactory;
45
use TYPO3\CMS\Core\Imaging\IconSize;
46
use TYPO3\CMS\Core\Utility\DebugUtility;
47
use TYPO3\CMS\Core\Utility\GeneralUtility;
48

49
/**
50
 * @internal since v12.0.0
51
 */
52
final class BackendModuleCrawlerLogController extends AbstractBackendModuleController implements BackendModuleControllerInterface
53
{
54
    public const BACKEND_MODULE = 'web_site_crawler_log';
55

56
    private QueryBuilder $queryBuilder;
57
    private bool $CSVExport = false;
58
    private readonly array $backendModuleMenu;
59
    private int $setId;
60
    private string $quiPath;
61
    private string $logDisplay;
62
    private int $itemsPerPage;
63
    private string $showResultLog;
64
    private string $showFeVars;
65
    private int $showSetId;
66
    private string $logDepth;
67
    /**
68
     * @var mixed|string|null
69
     */
70
    private mixed $queueId;
71

72
    public function __construct(
73
        private readonly QueueRepository $queueRepository,
74
        private readonly CsvWriterInterface $csvWriter,
75
        private readonly IconFactory $iconFactory,
76
        private readonly CrawlUrlMessageDispatcher $crawlUrlMessageDispatcher,
77
        private readonly BackendModuleLogService $backendModuleLogService,
78
        private readonly BackendModuleScriptUrlService $backendModuleScriptUrlService,
79
    ) {
80
        $this->backendModuleMenu = $this->getModuleMenu();
×
81
    }
82

83
    #[Required]
84
    public function setQueryBuilder(): void
85
    {
86
        $this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(
×
87
            QueueRepository::TABLE_NAME
×
88
        );
×
89
    }
90

91
    #[\Override]
92
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
93
    {
94
        $this->setPropertiesBasedOnPostVars($request);
×
95
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
96
        $this->moduleTemplate = $this->moduleTemplate->makeDocHeaderModuleMenu([
×
97
            'id' => $this->pageUid,
×
98
        ]);
×
99

100
        if (!$this->pageUid) {
×
101
            $this->moduleTemplate->assign('noPageSelected', true);
×
102
            return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
103
        }
104
        $this->moduleTemplate = $this->assignValues($request);
×
105
        return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
106
    }
107

108
    private function getQueueEntry(mixed $queueId): array
109
    {
110
        $q_entry = $this->queryBuilder
×
111
            ->from(QueueRepository::TABLE_NAME)
×
112
            ->select('*')->where(
×
113
                $this->queryBuilder->expr()->eq('qid', $this->queryBuilder->createNamedParameter($queueId))
×
114
            )->executeQuery()
×
115
            ->fetchAssociative();
×
116

117
        if ($q_entry === false) {
×
118
            return [[], ResultHandler::getResStatus([])];
×
119
        }
120

121
        // Explode values
122
        $q_entry['parameters'] = json_decode((string) $q_entry['parameters'], true);
×
123
        $q_entry['result_data'] = json_decode((string) $q_entry['result_data'], true);
×
124
        $resStatus = ResultHandler::getResStatus($q_entry['result_data']);
×
125
        if (is_array($q_entry['result_data'])) {
×
126
            $q_entry['result_data']['content'] = json_decode((string) $q_entry['result_data']['content'], true);
×
127
            if (!$this->showResultLog && is_array($q_entry['result_data']['content'])) {
×
128
                unset($q_entry['result_data']['content']['log']);
×
129
            }
130
        }
131
        return [$q_entry, $resStatus];
×
132
    }
133

134
    /**
135
     * @throws RouteNotFoundException
136
     */
137
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
138
    {
139
        // Look for set ID sent - if it is, we will display contents of that set:
140
        $this->showSetId = RequestHelper::getIntFromRequest($request, 'setID');
×
141
        $this->CSVExport = RequestHelper::getBoolFromRequest($request, '_csv');
×
142
        $logEntriesPerPage = [];
×
143
        $csvData = [];
×
144

145
        $quidRead = RequestHelper::getIntFromRequest($request, 'qid_read');
×
146
        if ($quidRead) {
×
147
            // Re-crawl an already crawled entry, requested by an editor and therefore with high priority
148
            $this->crawlUrlMessageDispatcher->dispatch(
×
149
                new CrawlUrlMessage($quidRead, true),
×
150
                priority: CrawlPriority::High
×
151
            );
×
152
        }
153

154
        if ($this->queueId) {
×
155
            // Get entry record:
156
            [$q_entry, $resStatus] = $this->getQueueEntry($this->queueId);
×
157
            $this->moduleTemplate->assignMultiple([
×
158
                'queueStatus' => $resStatus,
×
159
                'queueDetails' => DebugUtility::viewArray($q_entry),
×
160
            ]);
×
161
        } else {
162
            // Show list
163
            // Drawing tree:
164
            $tree = GeneralUtility::makeInstance(PageTreeView::class);
×
165
            $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
×
166
            $tree->init('AND ' . $perms_clause);
×
167

168
            // Set root row:
169
            $pageinfo = BackendUtility::readPageAccess($this->pageUid, $perms_clause);
×
170

171
            if (is_array($pageinfo)) {
×
172
                $HTML = $this->iconFactory->getIconForRecord('pages', $pageinfo, IconSize::SMALL)->render();
×
173
                $tree->tree[] = [
×
174
                    'row' => $pageinfo,
×
175
                    'HTML' => $HTML,
×
176
                ];
×
177
            }
178

179
            // Get branch beneath:
180
            if ($this->logDepth) {
×
181
                $tree->getTree($this->pageUid, (int) $this->logDepth);
×
182
            }
183

184
            // If Flush button is pressed, flush tables instead of selecting entries:
185
            $parsedBody = (array) ($request->getParsedBody() ?? []);
×
186
            if ($parsedBody['_flush'] ?? false) {
×
187
                $doFlush = true;
×
188
            } elseif ($parsedBody['_flush_all'] ?? false) {
×
189
                $doFlush = true;
×
190
                $this->logDisplay = 'all';
×
191
            } else {
192
                $doFlush = false;
×
193
            }
194

195
            $queueFilter = new QueueFilter($this->logDisplay);
×
196

197
            if ($doFlush) {
×
198
                $this->queueRepository->flushQueue($queueFilter);
×
199
            }
200

201
            // Traverse page tree:
202
            $count = 0;
×
203

204
            foreach ($tree->tree as $data) {
×
205
                $logEntriesOfPage = $this->queueRepository->getQueueEntriesForPageId(
×
206
                    (int) $data['row']['uid'],
×
207
                    $this->itemsPerPage,
×
208
                    $queueFilter
×
209
                );
×
210

NEW
211
                [$logEntriesPerPage[], $csvRows] = $this->backendModuleLogService->addRows(
×
212
                    $logEntriesOfPage,
×
213
                    RequestHelper::getIntFromRequest($request, 'setID'),
×
214
                    $data['HTML'] . BackendUtility::getRecordTitle('pages', $data['row'], true),
×
215
                    $this->showResultLog,
×
216
                    $this->showFeVars,
×
217
                    $this->CSVExport
×
218
                );
×
NEW
219
                array_push($csvData, ...$csvRows);
×
220

221
                if (++$count === 1000) {
×
222
                    break;
×
223
                }
224
            }
225

226
            $this->moduleTemplate->assign('logEntriesPerPage', $logEntriesPerPage);
×
227
        }
228

229
        if ($this->CSVExport) {
×
230
            $this->outputCsvFile($csvData);
×
231
        }
232

233
        $queryParams = [
×
234
            'setID' => $this->setId,
×
235
            'displayLog' => $this->logDisplay,
×
236
            'itemsPerPage' => $this->itemsPerPage,
×
237
            'ShowFeVars' => $this->showFeVars,
×
238
            'ShowResultLog' => $this->showResultLog,
×
239
            'logDepth' => $this->logDepth,
×
240
        ];
×
241

242
        return $this->moduleTemplate->assignMultiple([
×
243
            'backendModuleUrl' => $this->getBackendModuleUrl(),
×
244
            'actionUrl' => '',
×
245
            'queueId' => $this->queueId,
×
246
            'setId' => $this->showSetId,
×
247
            'noPageSelected' => false,
×
248
            'logEntriesPerPage' => $logEntriesPerPage,
×
249
            'showResultLog' => $this->showResultLog,
×
250
            'showFeVars' => $this->showFeVars,
×
251
            'displayActions' => 1,
×
252
            'displayLogFilterConfig' => [
×
253
                'name' => 'displayLog',
×
254
                'currentValue' => $this->logDisplay,
×
255
                'menuItems' => $this->backendModuleMenu['displayLog'],
×
256
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
257
                    $request,
×
258
                    'displayLog',
×
259
                    $this->pageUid,
×
260
                    $queryParams
×
261
                ),
×
262
            ],
×
263
            'itemPerPageConfig' => [
×
264
                'name' => 'itemsPerPage',
×
265
                'currentValue' => $this->itemsPerPage,
×
266
                'menuItems' => $this->backendModuleMenu['itemsPerPage'],
×
267
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
268
                    $request,
×
269
                    'itemsPerPage',
×
270
                    $this->pageUid,
×
271
                    $queryParams
×
272
                ),
×
273
            ],
×
274
            'showResultLogConfig' => [
×
275
                'name' => 'ShowResultLog',
×
276
                'currentValue' => $this->showResultLog,
×
277
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
278
                    $request,
×
279
                    'ShowResultLog',
×
280
                    $this->pageUid,
×
281
                    $queryParams,
×
282
                    $this->quiPath
×
283
                ),
×
284
            ],
×
285
            'showFeVarsConfig' => [
×
286
                'name' => 'ShowFeVars',
×
287
                'currentValue' => $this->showFeVars,
×
288
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
289
                    $request,
×
290
                    'ShowFeVars',
×
291
                    $this->pageUid,
×
292
                    $queryParams,
×
293
                    $this->quiPath
×
294
                ),
×
295
            ],
×
296
            'depthDropDownConfig' => [
×
297
                'name' => 'logDepth',
×
298
                'currentValue' => $this->logDepth,
×
299
                'menuItems' => $this->backendModuleMenu['logDepth'],
×
300
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
301
                    $request,
×
302
                    'logDepth',
×
303
                    $this->pageUid,
×
304
                    $queryParams
×
305
                ),
×
306
            ],
×
307
        ]);
×
308
    }
309

310
    private function setPropertiesBasedOnPostVars(ServerRequestInterface $request): void
311
    {
312
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
313
        $this->setId = RequestHelper::getIntFromRequest($request, 'setIDÍ„');
×
314
        $parsedBody = (array) ($request->getParsedBody() ?? []);
×
315
        $quidDetails = $parsedBody['qid_details'] ?? $request->getQueryParams()['qid_details'] ?? null;
×
316
        $this->quiPath = $quidDetails ? '&qid_details=' . (int) $quidDetails : '';
×
317
        $this->queueId = $quidDetails ?? null;
×
318
        $this->logDisplay = RequestHelper::getStringFromRequest($request, 'displayLog', 'all');
×
319
        $this->itemsPerPage = RequestHelper::getIntFromRequest($request, 'itemsPerPage', 10);
×
320
        $this->showResultLog = RequestHelper::getStringFromRequest($request, 'ShowResultLog', '0');
×
321
        $this->showFeVars = RequestHelper::getStringFromRequest($request, 'ShowFeVars', '0');
×
322
        $this->logDepth = RequestHelper::getStringFromRequest($request, 'logDepth', '0');
×
323
    }
324

325
    /**
326
     * Outputs the CSV file and sets the correct headers
327
     */
328
    private function outputCsvFile(array $csvData): void
329
    {
330
        if (!count($csvData)) {
×
331
            MessageUtility::addWarningMessage(
×
332
                $this->getLanguageService()->sL('crawler.messages:message.canNotExportEmptyQueueToCsvText')
×
333
            );
×
334
            return;
×
335
        }
336

337
        $csvString = $this->csvWriter->arrayToCsv($csvData);
×
338

339
        header('Content-Type: application/octet-stream');
×
340
        header('Content-Disposition: attachment; filename=CrawlerLog.csv');
×
341
        echo $csvString;
×
342

343
        exit;
×
344
    }
345

346
    private function getBackendModuleUrl(): UriInterface
347
    {
348
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
349
    }
350
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc