• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43585475383

14 Nov 2026 10:28AM UTC coverage: 67.649% (+0.02%) from 67.631%
43585475383

push

github

web-flow
!!! [TASK] Move the queue writing out of CrawlerController (#1335)

The new QueueEntryWriter creates the URLs of crawler configurations,
writes them to the queue and hands them over to the message bus. It
replaces CrawlerController::urlListFromUrlArray() (now
compileUrlList()), addUrl() and getCurrentTime(), together with the
$setID (now $setId), $registerQueueEntriesInternallyOnly,
$queueEntries and $urlList state they use.

BuildQueueCommand, QueueService and the "Start Crawling" backend module
use the writer directly. QueueService no longer needs the controller.

CrawlerController is now below the cognitive complexity limit, so the
PHPStan baseline is empty.

105 of 111 new or added lines in 5 files covered. (94.59%)

1240 of 1833 relevant lines covered (67.65%)

5.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.88
/Classes/Queue/QueueEntryWriter.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Queue;
6

7
/*
8
 * (c) 2026 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
23
use AOE\Crawler\Domain\Repository\QueueRepository;
24
use AOE\Crawler\Event\AfterUrlAddedToQueueEvent;
25
use AOE\Crawler\Queue\Message\CrawlUrlMessage;
26
use AOE\Crawler\Service\ProcessInstructionService;
27
use AOE\Crawler\Service\UrlService;
28
use Psr\EventDispatcher\EventDispatcherInterface;
29
use Psr\Http\Message\UriInterface;
30
use TYPO3\CMS\Backend\Utility\BackendUtility;
31
use TYPO3\CMS\Core\Utility\GeneralUtility;
32

33
/**
34
 * Creates the URLs of crawler configurations, writes them to the queue (tx_crawler_queue)
35
 * and hands them over to the message bus.
36
 *
37
 * Not final, so tests can replace getCurrentTime().
38
 *
39
 * @internal
40
 */
41
class QueueEntryWriter
42
{
43
    /**
44
     * The set id of the queue entries written in the current run
45
     */
46
    public int $setId = 0;
47

48
    /**
49
     * If set, the queue entries are only collected in $queueEntries and not stored in the database
50
     */
51
    public bool $registerQueueEntriesInternallyOnly = false;
52

53
    /**
54
     * The queue entries collected when $registerQueueEntriesInternallyOnly is set
55
     */
56
    public array $queueEntries = [];
57

58
    /**
59
     * The URLs with their scheduled time, fx. "[14-11-26 12:00] https://example.com/"
60
     *
61
     * @var list<string>
62
     */
63
    public array $urlList = [];
64

65
    private readonly array $extensionSettings;
66

67
    public function __construct(
68
        private readonly QueueRepository $queueRepository,
69
        private readonly EventDispatcherInterface $eventDispatcher,
70
        private readonly UrlService $urlService,
71
        private readonly ProcessInstructionService $processInstructionService,
72
        private readonly CrawlUrlMessageDispatcher $crawlUrlMessageDispatcher,
73
        ExtensionConfigurationProvider $extensionConfigurationProvider,
74
    ) {
75
        $this->extensionSettings = $extensionConfigurationProvider->getExtensionConfiguration();
16 ✔
76
    }
77

78
    /**
79
     * Creates the URLs of a configuration and, if $submitCrawlUrls is set, adds them to the queue.
80
     * See Web > Info module script + "indexed_search"'s crawler hook-client using this!
81
     *
82
     * @param array $configuration A configuration of the page, see PageConfigurationResolver
83
     * @param array $pageRow Page row
84
     * @param int $scheduledTime Unix time to schedule indexing to, typically time()
85
     * @param int $reqMinute Number of requests per minute (creates the interleave between requests)
86
     * @param bool $submitCrawlUrls If set, submits the URLs to queue
87
     * @param bool $downloadCrawlUrls If set (and $submitCrawlUrls is not) fills $downloadUrls with the URLs
88
     * @param array $duplicateTrack An id per URL, so URLs already handled in this run are not added again
89
     * @param array $downloadUrls Filled with the URLs, if $downloadCrawlUrls is set
90
     * @param array $incomingProcInstructions Processing instructions, if empty the URLs are not filtered by them
91
     * @return string List of URLs (meant for display in backend module)
92
     */
93
    public function compileUrlList(
94
        array $configuration,
95
        array $pageRow,
96
        int $scheduledTime,
97
        int $reqMinute,
98
        bool $submitCrawlUrls,
99
        bool $downloadCrawlUrls,
100
        array &$duplicateTrack,
101
        array &$downloadUrls,
102
        array $incomingProcInstructions
103
    ): string {
104
        if (!is_array($configuration['URLs'])) {
10 ✔
NEW
105
            return 'ERROR - no URL generated';
×
106
        }
107
        $urlLog = [];
10 ✔
108
        $pageId = (int) $pageRow['uid'];
10 ✔
109
        $configurationHash = $this->getConfigurationHash($configuration);
10 ✔
110
        $skipInnerCheck = $this->queueRepository->noUnprocessedQueueEntriesForPageWithConfigurationHashExist(
10 ✔
111
            $pageId,
10 ✔
112
            $configurationHash
10 ✔
113
        );
10 ✔
114

115
        foreach ($configuration['URLs'] as $urlQuery) {
10 ✔
116
            if (!$this->processInstructionService->isAllowed(
10 ✔
117
                $configuration['subCfg']['procInstrFilter'] ?? '',
10 ✔
118
                $incomingProcInstructions
10 ✔
119
            )) {
10 ✔
NEW
120
                continue;
×
121
            }
122
            $url = $this->urlService->getUrlFromPageAndQueryParameters(
10 ✔
123
                $pageId,
10 ✔
124
                $urlQuery,
10 ✔
125
                $configuration['subCfg']['baseUrl'] ?? null,
10 ✔
126
                (int) ($configuration['subCfg']['force_ssl'] ?? 0)
10 ✔
127
            );
10 ✔
128

129
            if (!$url instanceof UriInterface) {
10 ✔
NEW
130
                continue;
×
131
            }
132

133
            $url = (string) $url;
10 ✔
134

135
            // Create key by which to determine unique-ness:
136
            $uKey = $url . '|' . ($configuration['subCfg']['userGroups'] ?? '') . '|' . ($configuration['subCfg']['procInstrFilter'] ?? '');
10 ✔
137

138
            if (isset($duplicateTrack[$uKey])) {
10 ✔
139
                //if the url key is registered just display it and do not resubmit is
NEW
140
                $urlLog[] = '<em><span class="text-muted">' . htmlspecialchars($url) . '</span></em>';
×
141
            } else {
142
                // Scheduled time:
143
                $schTime = $scheduledTime + round(count($duplicateTrack) * (60 / $reqMinute));
10 ✔
144
                $schTime = intval($schTime / 60) * 60;
10 ✔
145
                $formattedDate = BackendUtility::datetime($schTime);
10 ✔
146
                $this->urlList[] = '[' . $formattedDate . '] ' . $url;
10 ✔
147
                $urlList = '[' . $formattedDate . '] ' . htmlspecialchars($url);
10 ✔
148

149
                // Submit for crawling!
150
                if ($submitCrawlUrls) {
10 ✔
151
                    $added = $this->addUrl(
9 ✔
152
                        $pageId,
9 ✔
153
                        $url,
9 ✔
154
                        $configuration['subCfg'],
9 ✔
155
                        $scheduledTime,
9 ✔
156
                        $configurationHash,
9 ✔
157
                        $skipInnerCheck
9 ✔
158
                    );
9 ✔
159
                    if ($added === false) {
9 ✔
160
                        $urlList .= ' (URL already existed)';
4 ✔
161
                    }
162
                } elseif ($downloadCrawlUrls) {
1 ✔
163
                    $downloadUrls[$url] = $url;
1 ✔
164
                }
165
                $urlLog[] = $urlList;
10 ✔
166
            }
167
            $duplicateTrack[$uKey] = true;
10 ✔
168
        }
169

170
        // Todo: Find a better option to have this correct in both backend (<br>) and cli (<new line>)
171
        return implode('<br>', $urlLog);
10 ✔
172
    }
173

174
    /**
175
     * Adds a URL to the queue, unless an equal entry is already queued. Returns whether it was added.
176
     *
177
     * @param array $subCfg Sub configuration array (from TS config)
178
     * @param int $scheduled Scheduled time
179
     * @param bool $skipInnerDuplicationCheck Skip the check for an equal entry
180
     */
181
    public function addUrl(
182
        int $pageId,
183
        string $url,
184
        array $subCfg,
185
        int $scheduled,
186
        string $configurationHash = '',
187
        bool $skipInnerDuplicationCheck = false
188
    ): bool {
189
        $urlAdded = false;
14 ✔
190
        $rows = [];
14 ✔
191

192
        // Creating parameters:
193
        $parameters = [
14 ✔
194
            'url' => $url,
14 ✔
195
        ];
14 ✔
196

197
        // fe user group simulation:
198
        $uGs = implode(',', array_unique(GeneralUtility::intExplode(',', $subCfg['userGroups'] ?? '', true)));
14 ✔
199
        if ($uGs) {
14 ✔
200
            $parameters['feUserGroupList'] = $uGs;
1 ✔
201
        }
202

203
        // Setting processing instructions
204
        $parameters['procInstructions'] = GeneralUtility::trimExplode(',', $subCfg['procInstrFilter'] ?? '');
14 ✔
205
        if (is_array($subCfg['procInstrParams.'] ?? false)) {
14 ✔
206
            $parameters['procInstrParams'] = $subCfg['procInstrParams.'];
10 ✔
207
        }
208

209
        // Compile value array:
210
        $parameters_serialized = json_encode($parameters) ?: '';
14 ✔
211
        $fieldArray = [
14 ✔
212
            'page_id' => $pageId,
14 ✔
213
            'parameters' => $parameters_serialized,
14 ✔
214
            'parameters_hash' => md5($parameters_serialized),
14 ✔
215
            'configuration_hash' => $configurationHash,
14 ✔
216
            'scheduled' => $scheduled,
14 ✔
217
            'exec_time' => 0,
14 ✔
218
            'set_id' => $this->setId,
14 ✔
219
            'result_data' => '',
14 ✔
220
            'configuration' => $subCfg['key'],
14 ✔
221
        ];
14 ✔
222

223
        if ($this->registerQueueEntriesInternallyOnly) {
14 ✔
224
            //the entries will only be registered and not stored to the database
225
            $this->queueEntries[] = $fieldArray;
3 ✔
226
        } else {
227
            if (!$skipInnerDuplicationCheck) {
11 ✔
228
                // check if there is already an equal entry
229
                $rows = $this->queueRepository->getDuplicateQueueItemsIfExists(
6 ✔
230
                    (bool) $this->extensionSettings['enableTimeslot'],
6 ✔
231
                    $scheduled,
6 ✔
232
                    $this->getCurrentTime(),
6 ✔
233
                    $fieldArray['page_id'],
6 ✔
234
                    $fieldArray['parameters_hash']
6 ✔
235
                );
6 ✔
236
            }
237
            if ($rows === []) {
11 ✔
238
                $uid = $this->queueRepository->add($fieldArray);
10 ✔
239
                $urlAdded = true;
10 ✔
240

241
                $this->eventDispatcher->dispatch(new AfterUrlAddedToQueueEvent((string) $uid, $fieldArray));
10 ✔
242
                $this->dispatchCrawlUrlMessage($uid, $pageId, $scheduled);
10 ✔
243
            }
244
        }
245

246
        return $urlAdded;
14 ✔
247
    }
248

249
    /**
250
     * Returns the current system time
251
     */
252
    public function getCurrentTime(): int
253
    {
254
        return time();
10 ✔
255
    }
256

257
    /**
258
     * Returns a md5 hash generated from a serialized configuration array.
259
     */
260
    protected function getConfigurationHash(array $configuration): string
261
    {
262
        unset($configuration['paramExpanded']);
16 ✔
263
        unset($configuration['URLs']);
16 ✔
264
        return md5(serialize($configuration));
16 ✔
265
    }
266

267
    /**
268
     * Hands the queue record over to the message bus. Records scheduled in the future are delayed accordingly.
269
     */
270
    private function dispatchCrawlUrlMessage(int $queueId, int $pageId, int $scheduled): void
271
    {
272
        $this->crawlUrlMessageDispatcher->dispatch(
10 ✔
273
            new CrawlUrlMessage($queueId),
10 ✔
274
            $pageId,
10 ✔
275
            $scheduled - $this->getCurrentTime()
10 ✔
276
        );
10 ✔
277
    }
278
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc