• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 11237471329

08 Oct 2024 02:20PM UTC coverage: 68.586% (-1.3%) from 69.862%
11237471329

push

github

web-flow
ci: Update coveralls workflow (#1109)

1834 of 2674 relevant lines covered (68.59%)

3.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleCrawlerLogController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\ResultHandler;
23
use AOE\Crawler\Controller\CrawlerController;
24
use AOE\Crawler\Converter\JsonCompatibilityConverter;
25
use AOE\Crawler\Domain\Repository\QueueRepository;
26
use AOE\Crawler\Service\BackendModuleLogService;
27
use AOE\Crawler\Service\BackendModuleScriptUrlService;
28
use AOE\Crawler\Utility\MessageUtility;
29
use AOE\Crawler\Value\QueueFilter;
30
use AOE\Crawler\Writer\FileWriter\CsvWriter\CsvWriterInterface;
31
use Psr\Http\Message\ResponseInterface;
32
use Psr\Http\Message\ServerRequestInterface;
33
use Symfony\Contracts\Service\Attribute\Required;
34
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
35
use TYPO3\CMS\Backend\Template\ModuleTemplate;
36
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
37
use TYPO3\CMS\Backend\Utility\BackendUtility;
38
use TYPO3\CMS\Core\Database\ConnectionPool;
39
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
40
use TYPO3\CMS\Core\Imaging\Icon;
41
use TYPO3\CMS\Core\Imaging\IconFactory;
42
use TYPO3\CMS\Core\Utility\DebugUtility;
43
use TYPO3\CMS\Core\Utility\GeneralUtility;
44

45
/**
46
 * @internal since v12.0.0
47
 */
48
final class BackendModuleCrawlerLogController extends AbstractBackendModuleController implements BackendModuleControllerInterface
49
{
50
    public const BACKEND_MODULE = 'web_site_crawler_log';
51

52
    private QueryBuilder $queryBuilder;
53
    private bool $CSVExport = false;
54
    private readonly array $backendModuleMenu;
55
    private int $setId;
56
    private string $quiPath;
57
    private string $logDisplay;
58
    private int $itemsPerPage;
59
    private string $showResultLog;
60
    private string $showFeVars;
61
    private int $showSetId;
62
    private string $logDepth;
63
    /**
64
     * @var mixed|string|null
65
     */
66
    private mixed $queueId;
67

68
    public function __construct(
69
        private readonly QueueRepository $queueRepository,
70
        private readonly CsvWriterInterface $csvWriter,
71
        private readonly JsonCompatibilityConverter $jsonCompatibilityConverter,
72
        private readonly IconFactory $iconFactory,
73
        private readonly CrawlerController $crawlerController,
74
        private readonly BackendModuleLogService $backendModuleLogService,
75
        private readonly BackendModuleScriptUrlService $backendModuleScriptUrlService,
76
    ) {
77
        $this->backendModuleMenu = $this->getModuleMenu();
×
78
    }
79

80
    #[Required]
81
    public function setQueryBuilder(): void
82
    {
83
        $this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(
×
84
            QueueRepository::TABLE_NAME
×
85
        );
×
86
    }
87

88
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
89
    {
90
        $this->setPropertiesBasedOnPostVars($request);
×
91
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
92
        $this->moduleTemplate = $this->moduleTemplate->makeDocHeaderModuleMenu([
×
93
            'id' => $this->pageUid,
×
94
        ]);
×
95

96
        if (!$this->pageUid) {
×
97
            $this->isErrorDetected = true;
×
98
            $this->moduleTemplate->assign('noPageSelected', true);
×
99
            return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
100
        }
101
        $this->moduleTemplate = $this->assignValues($request);
×
102
        return $this->moduleTemplate->renderResponse('Backend/ShowLog');
×
103
    }
104

105
    private function getQueueEntry(mixed $queueId): array
106
    {
107
        $q_entry = $this->queryBuilder
×
108
            ->from(QueueRepository::TABLE_NAME)
×
109
            ->select('*')->where(
×
110
                $this->queryBuilder->expr()->eq('qid', $this->queryBuilder->createNamedParameter($queueId))
×
111
            )->executeQuery()
×
112
            ->fetchAssociative();
×
113

114
        // Explode values
115
        $q_entry['parameters'] = $this->jsonCompatibilityConverter->convert($q_entry['parameters']);
×
116
        $q_entry['result_data'] = $this->jsonCompatibilityConverter->convert($q_entry['result_data']);
×
117
        $resStatus = ResultHandler::getResStatus($q_entry['result_data']);
×
118
        if (is_array($q_entry['result_data'])) {
×
119
            $q_entry['result_data']['content'] = $this->jsonCompatibilityConverter->convert(
×
120
                $q_entry['result_data']['content']
×
121
            );
×
122
            if (!$this->showResultLog) {
×
123
                if (is_array($q_entry['result_data']['content'])) {
×
124
                    unset($q_entry['result_data']['content']['log']);
×
125
                }
126
            }
127
        }
128
        return [$q_entry, $resStatus];
×
129
    }
130

131
    /**
132
     * @throws RouteNotFoundException
133
     */
134
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
135
    {
136
        // Look for set ID sent - if it is, we will display contents of that set:
137
        $this->showSetId = (int) ($request->getParsedBody()['setID'] ?? $request->getQueryParams()['setID'] ?? 0);
×
138
        $this->CSVExport = (bool) ($request->getParsedBody()['_csv'] ?? $request->getQueryParams()['_csv'] ?? false);
×
139
        $logEntriesPerPage = [];
×
140
        $csvData = [];
×
141

142
        $quidRead = (int) ($request->getParsedBody()['qid_read'] ?? $request->getQueryParams()['qid_read'] ?? 0);
×
143
        if ($quidRead) {
×
144
            $this->crawlerController->readUrl($quidRead, true);
×
145
        }
146

147
        if ($this->queueId) {
×
148
            // Get entry record:
149
            [$q_entry, $resStatus] = $this->getQueueEntry($this->queueId);
×
150
            $this->moduleTemplate->assignMultiple([
×
151
                'queueStatus' => $resStatus,
×
152
                'queueDetails' => DebugUtility::viewArray($q_entry),
×
153
            ]);
×
154
        } else {
155
            // Show list
156
            // Drawing tree:
157
            $tree = GeneralUtility::makeInstance(PageTreeView::class);
×
158
            $perms_clause = $GLOBALS['BE_USER']->getPagePermsClause(1);
×
159
            $tree->init('AND ' . $perms_clause);
×
160

161
            // Set root row:
162
            $pageinfo = BackendUtility::readPageAccess($this->pageUid, $perms_clause);
×
163

164
            if (is_array($pageinfo)) {
×
165
                $HTML = $this->iconFactory->getIconForRecord('pages', $pageinfo, Icon::SIZE_SMALL)->render();
×
166
                $tree->tree[] = [
×
167
                    'row' => $pageinfo,
×
168
                    'HTML' => $HTML,
×
169
                ];
×
170
            }
171

172
            // Get branch beneath:
173
            if ($this->logDepth) {
×
174
                $tree->getTree($this->pageUid, (int) $this->logDepth);
×
175
            }
176

177
            // If Flush button is pressed, flush tables instead of selecting entries:
178
            if ($request->getParsedBody()['_flush'] ?? false) {
×
179
                $doFlush = true;
×
180
            } elseif ($request->getParsedBody()['_flush_all'] ?? false) {
×
181
                $doFlush = true;
×
182
                $this->logDisplay = 'all';
×
183
            } else {
184
                $doFlush = false;
×
185
            }
186

187
            $queueFilter = new QueueFilter($this->logDisplay);
×
188

189
            if ($doFlush) {
×
190
                $this->queueRepository->flushQueue($queueFilter);
×
191
            }
192

193
            // Traverse page tree:
194
            $count = 0;
×
195

196
            foreach ($tree->tree as $data) {
×
197
                $logEntriesOfPage = $this->queueRepository->getQueueEntriesForPageId(
×
198
                    (int) $data['row']['uid'],
×
199
                    $this->itemsPerPage,
×
200
                    $queueFilter
×
201
                );
×
202

203
                [$logEntriesPerPage[], $row] = $this->backendModuleLogService->addRows(
×
204
                    $logEntriesOfPage,
×
205
                    (int) ($request->getParsedBody()['setID'] ?? $request->getQueryParams()['setID'] ?? 0),
×
206
                    $data['HTML'] . BackendUtility::getRecordTitle('pages', $data['row'], true),
×
207
                    $this->showResultLog,
×
208
                    $this->showFeVars,
×
209
                    $this->CSVExport
×
210
                );
×
211
                $csvData[] = $row;
×
212

213
                if (++$count === 1000) {
×
214
                    break;
×
215
                }
216
            }
217

218
            $this->moduleTemplate->assign('logEntriesPerPage', $logEntriesPerPage);
×
219
        }
220

221
        if ($this->CSVExport) {
×
222
            $this->outputCsvFile($csvData);
×
223
        }
224

225
        $queryParams = [
×
226
            'setID' => $this->setId,
×
227
            'displayLog' => $this->logDisplay,
×
228
            'itemsPerPage' => $this->itemsPerPage,
×
229
            'ShowFeVars' => $this->showFeVars,
×
230
            'ShowResultLog' => $this->showResultLog,
×
231
            'logDepth' => $this->logDepth,
×
232
        ];
×
233

234
        return $this->moduleTemplate->assignMultiple([
×
235
            'actionUrl' => '',
×
236
            'queueId' => $this->queueId,
×
237
            'setId' => $this->showSetId,
×
238
            'noPageSelected' => false,
×
239
            'logEntriesPerPage' => $logEntriesPerPage,
×
240
            'showResultLog' => $this->showResultLog,
×
241
            'showFeVars' => $this->showFeVars,
×
242
            'displayActions' => 1,
×
243
            'displayLogFilterConfig' => [
×
244
                'name' => 'displayLog',
×
245
                'currentValue' => $this->logDisplay,
×
246
                'menuItems' => $this->backendModuleMenu['displayLog'],
×
247
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
248
                    $request,
×
249
                    'displayLog',
×
250
                    $this->pageUid,
×
251
                    $queryParams
×
252
                ),
×
253
            ],
×
254
            'itemPerPageConfig' => [
×
255
                'name' => 'itemsPerPage',
×
256
                'currentValue' => $this->itemsPerPage,
×
257
                'menuItems' => $this->backendModuleMenu['itemsPerPage'],
×
258
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
259
                    $request,
×
260
                    'itemsPerPage',
×
261
                    $this->pageUid,
×
262
                    $queryParams
×
263
                ),
×
264
            ],
×
265
            'showResultLogConfig' => [
×
266
                'name' => 'ShowResultLog',
×
267
                'currentValue' => $this->showResultLog,
×
268
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
269
                    $request,
×
270
                    'ShowResultLog',
×
271
                    $this->pageUid,
×
272
                    $queryParams,
×
273
                    $this->quiPath
×
274
                ),
×
275
            ],
×
276
            'showFeVarsConfig' => [
×
277
                'name' => 'ShowFeVars',
×
278
                'currentValue' => $this->showFeVars,
×
279
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
280
                    $request,
×
281
                    'ShowFeVars',
×
282
                    $this->pageUid,
×
283
                    $queryParams,
×
284
                    $this->quiPath
×
285
                ),
×
286
            ],
×
287
            'depthDropDownConfig' => [
×
288
                'name' => 'logDepth',
×
289
                'currentValue' => $this->logDepth,
×
290
                'menuItems' => $this->backendModuleMenu['logDepth'],
×
291
                'scriptUrl' => $this->backendModuleScriptUrlService->buildScriptUrl(
×
292
                    $request,
×
293
                    'logDepth',
×
294
                    $this->pageUid,
×
295
                    $queryParams
×
296
                ),
×
297
            ],
×
298
        ]);
×
299
    }
300

301
    private function setPropertiesBasedOnPostVars(ServerRequestInterface $request): void
302
    {
303
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
304
        $this->setId = (int) ($request->getParsedBody()['setID'] ?? $request->getQueryParams()['setID'] ?? 0);
×
305
        $quidDetails = $request->getParsedBody()['qid_details'] ?? $request->getQueryParams()['qid_details'] ?? null;
×
306
        $this->quiPath = $quidDetails ? '&qid_details=' . (int) $quidDetails : '';
×
307
        $this->queueId = $quidDetails ?? null;
×
308
        $this->logDisplay = $request->getParsedBody()['displayLog'] ?? $request->getQueryParams()['displayLog'] ?? 'all';
×
309
        $this->itemsPerPage = (int) ($request->getParsedBody()['itemsPerPage'] ?? $request->getQueryParams()['itemsPerPage'] ?? 10);
×
310
        $this->showResultLog = (string) ($request->getParsedBody()['ShowResultLog'] ?? $request->getQueryParams()['ShowResultLog'] ?? 0);
×
311
        $this->showFeVars = (string) ($request->getParsedBody()['ShowFeVars'] ?? $request->getQueryParams()['ShowFeVars'] ?? 0);
×
312
        $this->logDepth = (string) ($request->getParsedBody()['logDepth'] ?? $request->getQueryParams()['logDepth'] ?? 0);
×
313
    }
314

315
    /**
316
     * Outputs the CSV file and sets the correct headers
317
     */
318
    private function outputCsvFile(array $csvData): void
319
    {
320
        if (!count($csvData)) {
×
321
            MessageUtility::addWarningMessage(
×
322
                $this->getLanguageService()->sL(
×
323
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:message.canNotExportEmptyQueueToCsvText'
×
324
                )
×
325
            );
×
326
            return;
×
327
        }
328

329
        $csvString = $this->csvWriter->arrayToCsv($csvData);
×
330

331
        header('Content-Type: application/octet-stream');
×
332
        header('Content-Disposition: attachment; filename=CrawlerLog.csv');
×
333
        echo $csvString;
×
334

335
        exit;
×
336
    }
337
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc