• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 19904598608

03 Dec 2025 06:29PM UTC coverage: 69.485% (-0.03%) from 69.514%
19904598608

Pull #1241

github

web-flow
Merge da108f5b5 into 15874a84e
Pull Request #1241: !!! [TASK] Remove class JsonCompatibilityConverter

10 of 14 new or added lines in 7 files covered. (71.43%)

1915 of 2756 relevant lines covered (69.48%)

3.17 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleCrawlerLogController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\ResultHandler;
24
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
25
use AOE\Crawler\Controller\CrawlerController;
26
use AOE\Crawler\Domain\Repository\QueueRepository;
27
use AOE\Crawler\Service\BackendModuleLogService;
28
use AOE\Crawler\Service\BackendModuleScriptUrlService;
29
use AOE\Crawler\Utility\MessageUtility;
30
use AOE\Crawler\Value\QueueFilter;
31
use AOE\Crawler\Writer\FileWriter\CsvWriter\CsvWriterInterface;
32
use Psr\Http\Message\ResponseInterface;
33
use Psr\Http\Message\ServerRequestInterface;
34
use Psr\Http\Message\UriInterface;
35
use Symfony\Contracts\Service\Attribute\Required;
36
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
37
use TYPO3\CMS\Backend\Template\ModuleTemplate;
38
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
39
use TYPO3\CMS\Backend\Utility\BackendUtility;
40
use TYPO3\CMS\Core\Database\ConnectionPool;
41
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
42
use TYPO3\CMS\Core\Imaging\IconFactory;
43
use TYPO3\CMS\Core\Imaging\IconSize;
44
use TYPO3\CMS\Core\Utility\DebugUtility;
45
use TYPO3\CMS\Core\Utility\GeneralUtility;
46

47
/**
48
 * @internal since v12.0.0
49
 */
50
final class BackendModuleCrawlerLogController extends AbstractBackendModuleController implements BackendModuleControllerInterface
51
{
52
    public const BACKEND_MODULE = 'web_site_crawler_log';
53

54
    private QueryBuilder $queryBuilder;
55
    private bool $CSVExport = false;
56
    private readonly array $backendModuleMenu;
57
    private int $setId;
58
    private string $quiPath;
59
    private string $logDisplay;
60
    private int $itemsPerPage;
61
    private string $showResultLog;
62
    private string $showFeVars;
63
    private int $showSetId;
64
    private string $logDepth;
65
    /**
66
     * @var mixed|string|null
67
     */
68
    private mixed $queueId;
69

70
    public function __construct(
71
        private readonly QueueRepository $queueRepository,
72
        private readonly CsvWriterInterface $csvWriter,
73
        private readonly IconFactory $iconFactory,
74
        private readonly CrawlerController $crawlerController,
75
        private readonly BackendModuleLogService $backendModuleLogService,
76
        private readonly BackendModuleScriptUrlService $backendModuleScriptUrlService,
77
    ) {
78
        $this->backendModuleMenu = $this->getModuleMenu();
×
79
    }
80

81
    #[Required]
82
    public function setQueryBuilder(): void
83
    {
84
        $this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(
×
85
            QueueRepository::TABLE_NAME
×
86
        );
×
87
    }
88

89
    #[\Override]
90
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
91
    {
92
        $this->setPropertiesBasedOnPostVars($request);
×
93
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
94
        $this->moduleTemplate = $this->moduleTemplate->makeDocHeaderModuleMenu([
×
95
            'id' => $this->pageUid,
×
96
        ]);
×
97

98
        if (!$this->pageUid) {
×
99
            $this->isErrorDetected = true;
×
100
            $this->moduleTemplate->assign('noPageSelected', true);
×
101
            return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
102
        }
103
        $this->moduleTemplate = $this->assignValues($request);
×
104
        return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
105
    }
106

107
    private function getQueueEntry(mixed $queueId): array
108
    {
109
        $q_entry = $this->queryBuilder
×
110
            ->from(QueueRepository::TABLE_NAME)
×
111
            ->select('*')->where(
×
112
                $this->queryBuilder->expr()->eq('qid', $this->queryBuilder->createNamedParameter($queueId))
×
113
            )->executeQuery()
×
114
            ->fetchAssociative();
×
115

116
        // Explode values
NEW
117
        $q_entry['parameters'] = json_decode((string) $q_entry['parameters'], true);
×
NEW
118
        $q_entry['result_data'] = json_decode((string) $q_entry['result_data'], true);
×
119
        $resStatus = ResultHandler::getResStatus($q_entry['result_data']);
×
120
        if (is_array($q_entry['result_data'])) {
×
NEW
121
            $q_entry['result_data']['content'] = json_decode((string) $q_entry['result_data']['content'], true);
×
122
            if (!$this->showResultLog) {
×
123
                if (is_array($q_entry['result_data']['content'])) {
×
124
                    unset($q_entry['result_data']['content']['log']);
×
125
                }
126
            }
127
        }
128
        return [$q_entry, $resStatus];
×
129
    }
130

131
    /**
132
     * @throws RouteNotFoundException
133
     */
134
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
135
    {
136
        // Look for set ID sent - if it is, we will display contents of that set:
137
        $this->showSetId = RequestHelper::getIntFromRequest($request, 'setID');
×
138
        $this->CSVExport = RequestHelper::getBoolFromRequest($request, '_csv');
×
139
        $logEntriesPerPage = [];
×
140
        $csvData = [];
×
141

142
        $quidRead = RequestHelper::getIntFromRequest($request, 'qid_read');
×
143
        if ($quidRead) {
×
144
            $this->crawlerController->readUrl($quidRead, true);
×
145
        }
146

147
        if ($this->queueId) {
×
148
            // Get entry record:
149
            [$q_entry, $resStatus] = $this->getQueueEntry($this->queueId);
×
150
            $this->moduleTemplate->assignMultiple([
×
151
                'queueStatus' => $resStatus,
×
152
                'queueDetails' => DebugUtility::viewArray($q_entry),
×
153
            ]);
×
154
        } else {
155
            // Show list
156
            // Drawing tree:
157
            $tree = GeneralUtility::makeInstance(PageTreeView::class);
×
158
            $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
×
159
            $tree->init('AND ' . $perms_clause);
×
160

161
            // Set root row:
162
            $pageinfo = BackendUtility::readPageAccess($this->pageUid, $perms_clause);
×
163

164
            if (is_array($pageinfo)) {
×
165
                $HTML = $this->iconFactory->getIconForRecord('pages', $pageinfo, IconSize::SMALL)->render();
×
166
                $tree->tree[] = [
×
167
                    'row' => $pageinfo,
×
168
                    'HTML' => $HTML,
×
169
                ];
×
170
            }
171

172
            // Get branch beneath:
173
            if ($this->logDepth) {
×
174
                $tree->getTree($this->pageUid, (int) $this->logDepth);
×
175
            }
176

177
            // If Flush button is pressed, flush tables instead of selecting entries:
178
            if ($request->getParsedBody()['_flush'] ?? false) {
×
179
                $doFlush = true;
×
180
            } elseif ($request->getParsedBody()['_flush_all'] ?? false) {
×
181
                $doFlush = true;
×
182
                $this->logDisplay = 'all';
×
183
            } else {
184
                $doFlush = false;
×
185
            }
186

187
            $queueFilter = new QueueFilter($this->logDisplay);
×
188

189
            if ($doFlush) {
×
190
                $this->queueRepository->flushQueue($queueFilter);
×
191
            }
192

193
            // Traverse page tree:
194
            $count = 0;
×
195

196
            foreach ($tree->tree as $data) {
×
197
                $logEntriesOfPage = $this->queueRepository->getQueueEntriesForPageId(
×
198
                    (int) $data['row']['uid'],
×
199
                    $this->itemsPerPage,
×
200
                    $queueFilter
×
201
                );
×
202

203
                [$logEntriesPerPage[], $row] = $this->backendModuleLogService->addRows(
×
204
                    $logEntriesOfPage,
×
205
                    RequestHelper::getIntFromRequest($request, 'setID'),
×
206
                    $data['HTML'] . BackendUtility::getRecordTitle('pages', $data['row'], true),
×
207
                    $this->showResultLog,
×
208
                    $this->showFeVars,
×
209
                    $this->CSVExport
×
210
                );
×
211
                $csvData[] = $row;
×
212

213
                if (++$count === 1000) {
×
214
                    break;
×
215
                }
216
            }
217

218
            $this->moduleTemplate->assign('logEntriesPerPage', $logEntriesPerPage);
×
219
        }
220

221
        if ($this->CSVExport) {
×
222
            $this->outputCsvFile($csvData);
×
223
        }
224

225
        $queryParams = [
×
226
            'setID' => $this->setId,
×
227
            'displayLog' => $this->logDisplay,
×
228
            'itemsPerPage' => $this->itemsPerPage,
×
229
            'ShowFeVars' => $this->showFeVars,
×
230
            'ShowResultLog' => $this->showResultLog,
×
231
            'logDepth' => $this->logDepth,
×
232
        ];
×
233

234
        return $this->moduleTemplate->assignMultiple([
×
235
            'backendModuleUrl' => $this->getBackendModuleUrl(),
×
236
            'actionUrl' => '',
×
237
            'queueId' => $this->queueId,
×
238
            'setId' => $this->showSetId,
×
239
            'noPageSelected' => false,
×
240
            'logEntriesPerPage' => $logEntriesPerPage,
×
241
            'showResultLog' => $this->showResultLog,
×
242
            'showFeVars' => $this->showFeVars,
×
243
            'displayActions' => 1,
×
244
            'displayLogFilterConfig' => [
×
245
                'name' => 'displayLog',
×
246
                'currentValue' => $this->logDisplay,
×
247
                'menuItems' => $this->backendModuleMenu['displayLog'],
×
248
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
249
                    $request,
×
250
                    'displayLog',
×
251
                    $this->pageUid,
×
252
                    $queryParams
×
253
                ),
×
254
            ],
×
255
            'itemPerPageConfig' => [
×
256
                'name' => 'itemsPerPage',
×
257
                'currentValue' => $this->itemsPerPage,
×
258
                'menuItems' => $this->backendModuleMenu['itemsPerPage'],
×
259
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
260
                    $request,
×
261
                    'itemsPerPage',
×
262
                    $this->pageUid,
×
263
                    $queryParams
×
264
                ),
×
265
            ],
×
266
            'showResultLogConfig' => [
×
267
                'name' => 'ShowResultLog',
×
268
                'currentValue' => $this->showResultLog,
×
269
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
270
                    $request,
×
271
                    'ShowResultLog',
×
272
                    $this->pageUid,
×
273
                    $queryParams,
×
274
                    $this->quiPath
×
275
                ),
×
276
            ],
×
277
            'showFeVarsConfig' => [
×
278
                'name' => 'ShowFeVars',
×
279
                'currentValue' => $this->showFeVars,
×
280
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
281
                    $request,
×
282
                    'ShowFeVars',
×
283
                    $this->pageUid,
×
284
                    $queryParams,
×
285
                    $this->quiPath
×
286
                ),
×
287
            ],
×
288
            'depthDropDownConfig' => [
×
289
                'name' => 'logDepth',
×
290
                'currentValue' => $this->logDepth,
×
291
                'menuItems' => $this->backendModuleMenu['logDepth'],
×
292
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
293
                    $request,
×
294
                    'logDepth',
×
295
                    $this->pageUid,
×
296
                    $queryParams
×
297
                ),
×
298
            ],
×
299
        ]);
×
300
    }
301

302
    private function setPropertiesBasedOnPostVars(ServerRequestInterface $request): void
303
    {
304
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
305
        $this->setId = RequestHelper::getIntFromRequest($request, 'setIDÍ„');
×
306
        $quidDetails = $request->getParsedBody()['qid_details'] ?? $request->getQueryParams()['qid_details'] ?? null;
×
307
        $this->quiPath = $quidDetails ? '&qid_details=' . (int) $quidDetails : '';
×
308
        $this->queueId = $quidDetails ?? null;
×
309
        $this->logDisplay = RequestHelper::getStringFromRequest($request, 'displayLog', 'all');
×
310
        $this->itemsPerPage = RequestHelper::getIntFromRequest($request, 'itemsPerPage', 10);
×
311
        $this->showResultLog = RequestHelper::getStringFromRequest($request, 'ShowResultLog', '0');
×
312
        $this->showFeVars = RequestHelper::getStringFromRequest($request, 'ShowFeVars', '0');
×
313
        $this->logDepth = RequestHelper::getStringFromRequest($request, 'logDepth', '0');
×
314
    }
315

316
    /**
317
     * Outputs the CSV file and sets the correct headers
318
     */
319
    private function outputCsvFile(array $csvData): void
320
    {
321
        if (!count($csvData)) {
×
322
            MessageUtility::addWarningMessage(
×
323
                $this->getLanguageService()->sL(
×
324
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:message.canNotExportEmptyQueueToCsvText'
×
325
                )
×
326
            );
×
327
            return;
×
328
        }
329

330
        $csvString = $this->csvWriter->arrayToCsv($csvData);
×
331

332
        header('Content-Type: application/octet-stream');
×
333
        header('Content-Disposition: attachment; filename=CrawlerLog.csv');
×
334
        echo $csvString;
×
335

336
        exit;
×
337
    }
338

339
    private function getBackendModuleUrl(): UriInterface
340
    {
341
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
342
    }
343
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc