• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43585475383

14 Nov 2026 10:28AM UTC coverage: 67.649% (+0.02%) from 67.631%
43585475383

push

github

web-flow
!!! [TASK] Move the queue writing out of CrawlerController (#1335)

The new QueueEntryWriter creates the URLs of crawler configurations,
writes them to the queue and hands them over to the message bus. It
replaces CrawlerController::urlListFromUrlArray() (now
compileUrlList()), addUrl() and getCurrentTime(), together with the
$setID (now $setId), $registerQueueEntriesInternallyOnly,
$queueEntries and $urlList state they use.

BuildQueueCommand, QueueService and the "Start Crawling" backend module
use the writer directly. QueueService no longer needs the controller.

CrawlerController is now below the cognitive complexity limit, so the
PHPStan baseline is empty.

105 of 111 new or added lines in 5 files covered. (94.59%)

1240 of 1833 relevant lines covered (67.65%)

5.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.1
/Classes/Command/BuildQueueCommand.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Command;
6

7
/*
8
 * (c) 2021 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
23
use AOE\Crawler\Controller\CrawlerController;
24
use AOE\Crawler\Domain\Model\Reason;
25
use AOE\Crawler\Domain\Repository\QueueRepository;
26
use AOE\Crawler\Event\InvokeQueueChangeEvent;
27
use AOE\Crawler\Queue\Handler\CrawlUrlHandler;
28
use AOE\Crawler\Queue\Message\CrawlUrlMessage;
29
use AOE\Crawler\Queue\QueueEntryWriter;
30
use AOE\Crawler\Utility\MessageUtility;
31
use AOE\Crawler\Value\QueueRow;
32
use Symfony\Component\Console\Attribute\AsCommand;
33
use Symfony\Component\Console\Command\Command;
34
use Symfony\Component\Console\Helper\ProgressBar;
35
use Symfony\Component\Console\Input\InputArgument;
36
use Symfony\Component\Console\Input\InputInterface;
37
use Symfony\Component\Console\Input\InputOption;
38
use Symfony\Component\Console\Output\OutputInterface;
39
use TYPO3\CMS\Core\Domain\Repository\PageRepository;
40
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
41
use TYPO3\CMS\Core\Utility\GeneralUtility;
42
use TYPO3\CMS\Core\Utility\MathUtility;
43

44
/**
45
 * @internal since v12.0.0
46
 */
47
#[AsCommand(name: 'crawler:buildQueue', description: 'Create entries in the queue that can be processed at once',)]
48
class BuildQueueCommand extends Command
49
{
50
    public function __construct(
51
        private readonly EventDispatcher $eventDispatcher,
52
        private readonly QueueRepository $queueRepository,
53
        private readonly PageRepository $pageRepository,
54
        private readonly CrawlerController $crawlerController,
55
        private readonly QueueEntryWriter $queueEntryWriter,
56
        private readonly CrawlUrlHandler $crawlUrlHandler,
57
    ) {
58
        parent::__construct();
8 ✔
59
    }
60

61
    #[\Override]
62
    protected function configure(): void
63
    {
64
        $this->setDescription('Create entries in the queue that can be processed at once');
8 ✔
65

66
        $this->setHelp(
8 ✔
67
            'Try "typo3 help crawler:buildQueue" to see your options' . chr(10) . chr(10) .
8 ✔
68
            'Works as a CLI interface to some functionality from the Web > Info > Site Crawler module;
8 ✔
69
It can put entries in the queue from command line options, return the list of URLs and even execute
70
all entries right away without having to queue them up - this can be useful for immediate re-cache,
71
re-indexing or static publishing from command line.' . chr(10) . chr(10) .
8 ✔
72
            '
8 ✔
73
            Examples:
74
              --- Re-cache pages from page 7 and two levels down, executed immediately
75
              $ typo3 crawler:buildQueue 7 defaultConfiguration --depth 2 --mode exec
76

77
              --- Put entries for re-caching pages from page 7 into queue, 4 every minute.
78
              $ typo3 crawler:buildQueue 7 defaultConfiguration --depth 0 --mode queue --number 4
79
            '
8 ✔
80
        );
8 ✔
81

82
        $this->addArgument('page', InputArgument::REQUIRED, 'The page from where the queue building should start');
8 ✔
83

84
        $this->addArgument('conf', InputArgument::REQUIRED, 'A comma separated list of crawler configurations');
8 ✔
85

86
        $this->addOption(
8 ✔
87
            'depth',
8 ✔
88
            'd',
8 ✔
89
            InputOption::VALUE_OPTIONAL,
8 ✔
90
            'Tree depth, 0-99\', "How many levels under the \'page_id\' to include.',
8 ✔
91
            '0'
8 ✔
92
        );
8 ✔
93

94
        $this->addOption(
8 ✔
95
            'mode',
8 ✔
96
            'm',
8 ✔
97
            InputOption::VALUE_OPTIONAL,
8 ✔
98
            'Specifies output modes url : Will list URLs which wget could use as input. queue: Will put entries in the queue, to be processed by "typo3 messenger:consume crawler_high crawler crawler_low". exec: Will execute all entries right away!'
8 ✔
99
        );
8 ✔
100

101
        $this->addOption(
8 ✔
102
            'number',
8 ✔
103
            '',
8 ✔
104
            InputOption::VALUE_OPTIONAL,
8 ✔
105
            'Specifies how many items are put in the queue per minute. Only valid for output mode "queue"',
8 ✔
106
            '0'
8 ✔
107
        );
8 ✔
108
    }
109

110
    /**
111
     * Crawler Command - Submitting URLs to be crawled.
112
     *
113
     * Works as a CLI interface to some functionality from the Web > Info > Site Crawler module;
114
     * It can put entries in the queue from command line options, return the list of URLs and even execute
115
     * all entries right away without having to queue them up - this can be useful for immediate re-cache,
116
     * re-indexing or static publishing from command line.
117
     *
118
     * Examples:
119
     *
120
     * --- Re-cache pages from page 7 and two levels down, executed immediately
121
     * $ typo3 crawler:buildQueue 7 defaultConfiguration --depth 2 --mode exec
122
     *
123
     *
124
     * --- Put entries for re-caching pages from page 7 into queue, 4 every minute.
125
     * $ typo3 crawler:buildQueue 7 defaultConfiguration --depth 0 --mode queue --number 4
126
     */
127
    #[\Override]
128
    protected function execute(InputInterface $input, OutputInterface $output): int
129
    {
130
        $mode = $input->getOption('mode') ?? 'queue';
8 ✔
131

132
        $extensionSettings = GeneralUtility::makeInstance(
8 ✔
133
            ExtensionConfigurationProvider::class
8 ✔
134
        )->getExtensionConfiguration();
8 ✔
135

136
        if ($mode === 'exec') {
8 ✔
137
            $this->queueEntryWriter->registerQueueEntriesInternallyOnly = true;
2 ✔
138
        }
139

140
        $pageId = MathUtility::forceIntegerInRange((int) $input->getArgument('page'), 0);
8 ✔
141
        if ($pageId === 0 || empty($this->pageRepository->getPage($pageId))) {
8 ✔
142
            $message = "Page {$pageId} is not a valid page, please check you root page id and try again.";
1 ✔
143
            MessageUtility::addErrorMessage($message);
1 ✔
144
            $output->writeln("<info>{$message}</info>");
1 ✔
145
            return Command::FAILURE;
1 ✔
146
        }
147

148
        $configurationKeys = $this->getConfigurationKeys((string) $input->getArgument('conf'));
7 ✔
149

150
        if ($mode === 'queue' || $mode === 'exec') {
7 ✔
151
            $reason = new Reason();
6 ✔
152
            $reason->setReason(Reason::REASON_CLI_SUBMIT);
6 ✔
153
            $reason->setDetailText('The cli script of the crawler added to the queue');
6 ✔
154
            $this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
6 ✔
155
        }
156

157
        if ($extensionSettings['cleanUpOldQueueEntries']) {
7 ✔
158
            $this->queueRepository->cleanUpOldQueueEntries();
7 ✔
159
        }
160

161
        $this->queueEntryWriter->setId = GeneralUtility::md5int(microtime());
7 ✔
162
        $queueRows = $this->getQueueRows($pageId, $input, $mode, $configurationKeys);
7 ✔
163

164
        match ($mode) {
7 ✔
165
            'url' => $output->writeln(
1 ✔
166
                '<info>' . implode(PHP_EOL, $this->crawlerController->downloadUrls) . PHP_EOL . '</info>'
1 ✔
167
            ),
1 ✔
168
            'exec' => $this->outputModeExec($output, $queueRows),
2 ✔
169
            'queue' => $this->outputModeQueue($output, $queueRows),
4 ✔
170
            default => $this->outputModeDefault($output, $queueRows),
×
171
        };
7 ✔
172

173
        return Command::SUCCESS;
7 ✔
174
    }
175

176
    /**
177
     * Obtains configuration keys from the CLI arguments
178
     */
179
    private function getConfigurationKeys(string $conf): array
180
    {
181
        $parameter = trim($conf);
7 ✔
182
        return $parameter !== '' ? GeneralUtility::trimExplode(',', $parameter) : [];
7 ✔
183
    }
184

185
    private function outputUrls(array $queueRows, OutputInterface $output): void
186
    {
187
        /** @var QueueRow $row */
188
        foreach ($queueRows as $row) {
6 ✔
189
            if (empty($row->message)) {
6 ✔
190
                $output->writeln('<info>' . $row->urls . '</info>');
6 ✔
191
            } else {
192
                $output->writeln(
1 ✔
193
                    '<comment>Page "' . $row->pageTitle . '" is not added to queue' . $row->message . '</comment>'
1 ✔
194
                );
1 ✔
195
            }
196
        }
197
    }
198

199
    private function outputModeDefault(OutputInterface $output, array $queueRows): void
200
    {
201
        $output->writeln(
×
202
            '<info>' . count(
×
NEW
203
                $this->queueEntryWriter->urlList
×
204
            ) . ' entries found for processing. (Use "mode" to decide action):</info>' . PHP_EOL
×
205
        );
×
206
        $this->outputUrls($queueRows, $output);
×
207
    }
208

209
    private function outputModeQueue(OutputInterface $output, array $queueRows): void
210
    {
211
        $output->writeln(
4 ✔
212
            '<info>Putting ' . count($this->queueEntryWriter->urlList) . ' entries in queue:</info>' . PHP_EOL
4 ✔
213
        );
4 ✔
214
        $this->outputUrls($queueRows, $output);
4 ✔
215
    }
216

217
    private function outputModeExec(OutputInterface $output, array $queueRows): void
218
    {
219
        $progressBar = new ProgressBar($output);
2 ✔
220
        $output->writeln(
2 ✔
221
            '<info>Executing ' . count($this->queueEntryWriter->urlList) . ' requests right away:</info>'
2 ✔
222
        );
2 ✔
223
        $this->outputUrls($queueRows, $output);
2 ✔
224
        $output->writeln('<info>Processing</info>' . PHP_EOL);
2 ✔
225

226
        foreach ($progressBar->iterate($this->queueEntryWriter->queueEntries) as $queueRec) {
2 ✔
227
            $p = json_decode((string) $queueRec['parameters'], true);
2 ✔
228
            if (is_bool($p)) {
2 ✔
229
                continue;
×
230
            }
231

232
            $progressBar->clear();
2 ✔
233
            if (empty($p['procInstructions'][0])) {
2 ✔
234
                $procInstructionsString = '';
2 ✔
235
            } else {
236
                $procInstructionsString = ' (' . implode(',', $p['procInstructions']) . ')';
×
237
            }
238
            $output->writeln('<info>' . $p['url'] . $procInstructionsString . ' => ' . '</info>');
2 ✔
239
            $progressBar->display();
2 ✔
240

241
            // Bypass the message queue and crawl the entry right away
242
            $result = ($this->crawlUrlHandler)(new CrawlUrlMessage($this->queueRepository->add($queueRec)));
2 ✔
243

244
            $resultContent = $result['content'] ?? '';
2 ✔
245
            $requestResult = json_decode((string) $resultContent, true);
2 ✔
246

247
            $progressBar->clear();
2 ✔
248
            if (is_array($requestResult)) {
2 ✔
249
                $resLog = array_key_exists('log', $requestResult)
2 ✔
250
                && is_array($requestResult['log']) ? chr(9) . chr(9) .
2 ✔
251
                    implode(PHP_EOL . chr(9) . chr(9), $requestResult['log']) : '';
2 ✔
252
                $output->writeln('<info>OK: ' . $resLog . '</info>' . PHP_EOL);
2 ✔
253
            } else {
254
                $output->writeln(
×
255
                    '<error>Error checking Crawler Result:  ' . substr(
×
256
                        (string) preg_replace('/\s+/', ' ', strip_tags((string) $resultContent)),
×
257
                        0,
×
258
                        30000
×
259
                    ) . '...' . PHP_EOL . '</error>' . PHP_EOL
×
260
                );
×
261
            }
262
            $progressBar->display();
2 ✔
263
        }
264
        $output->writeln('');
2 ✔
265
    }
266

267
    private function getQueueRows(int $pageId, InputInterface $input, mixed $mode, array $configurationKeys): array
268
    {
269
        return $this->crawlerController->getPageTreeAndUrls(
7 ✔
270
            $pageId,
7 ✔
271
            MathUtility::forceIntegerInRange((int) $input->getOption('depth'), 0, 99),
7 ✔
272
            $this->queueEntryWriter->getCurrentTime(),
7 ✔
273
            MathUtility::forceIntegerInRange((int) $input->getOption('number') ?: 30, 1, 1000),
7 ✔
274
            $mode === 'queue' || $mode === 'exec',
7 ✔
275
            $mode === 'url',
7 ✔
276
            [],
7 ✔
277
            $configurationKeys
7 ✔
278
        );
7 ✔
279
    }
280
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc