• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43585475383

14 Nov 2026 10:28AM UTC coverage: 67.649% (+0.02%) from 67.631%
43585475383

push

github

web-flow
!!! [TASK] Move the queue writing out of CrawlerController (#1335)

The new QueueEntryWriter creates the URLs of crawler configurations,
writes them to the queue and hands them over to the message bus. It
replaces CrawlerController::urlListFromUrlArray() (now
compileUrlList()), addUrl() and getCurrentTime(), together with the
$setID (now $setId), $registerQueueEntriesInternallyOnly,
$queueEntries and $urlList state they use.

BuildQueueCommand, QueueService and the "Start Crawling" backend module
use the writer directly. QueueService no longer needs the controller.

CrawlerController is now below the cognitive complexity limit, so the
PHPStan baseline is empty.

105 of 111 new or added lines in 5 files covered. (94.59%)

1240 of 1833 relevant lines covered (67.65%)

5.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleStartCrawlingController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
24
use AOE\Crawler\Controller\CrawlerController;
25
use AOE\Crawler\Domain\Model\Reason;
26
use AOE\Crawler\Event\InvokeQueueChangeEvent;
27
use AOE\Crawler\Queue\QueueEntryWriter;
28
use AOE\Crawler\Service\PageConfigurationResolver;
29
use AOE\Crawler\Utility\MessageUtility;
30
use Psr\Http\Message\ResponseInterface;
31
use Psr\Http\Message\ServerRequestInterface;
32
use Psr\Http\Message\UriInterface;
33
use Symfony\Contracts\Service\Attribute\Required;
34
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
35
use TYPO3\CMS\Backend\Routing\UriBuilder;
36
use TYPO3\CMS\Backend\Template\ModuleTemplate;
37
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
38
use TYPO3\CMS\Core\Utility\GeneralUtility;
39

40
/**
41
 * @internal since v12.0.0
42
 */
43
class BackendModuleStartCrawlingController extends AbstractBackendModuleController implements BackendModuleControllerInterface
44
{
45
    private const BACKEND_MODULE = 'web_site_crawler_start';
46
    private const LINE_FEED = 10;
47
    private const CARRIAGE_RETURN = 13;
48
    private int $reqMinute = 1000;
49
    private EventDispatcher $eventDispatcher;
50

51
    /**
52
     * @var array holds the selection of configuration from the configuration selector box
53
     */
54
    private $incomingConfigurationSelection = [];
55

56
    public function __construct(
57
        private readonly CrawlerController $crawlerController,
58
        private readonly PageConfigurationResolver $pageConfigurationResolver,
59
        private readonly QueueEntryWriter $queueEntryWriter,
60
        private readonly UriBuilder $backendUriBuilder,
61
    ) {
62
    }
×
63

64
    #[Required]
65
    public function setEventDispatcher(): void
66
    {
67
        $this->eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);
×
68
    }
69

70
    #[\Override]
71
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
72
    {
73
        $this->checkPhpBinaryForDirectRequests($this->extensionSettings);
×
74
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
75
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
76
        $this->moduleTemplate->makeDocHeaderModuleMenu([
×
77
            'id' => $request->getQueryParams()['id'] ?? -1,
×
78
        ]);
×
79

80
        $this->assignValues($request);
×
81
        return $this->moduleTemplate->renderResponse('Backend/ShowCrawlerInformation');
×
82
    }
83

84
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
85
    {
86
        $logUrl = $this->backendUriBuilder->buildUriFromRoute('web_site_crawler_log', [
×
87
            'id' => $this->pageUid,
×
88
        ]);
×
89

90
        $crawlingDepth = RequestHelper::getIntFromRequest($request, 'crawlingDepth');
×
91
        $submitCrawlUrls = RequestHelper::getBoolFromRequest($request, '_crawl');
×
92
        $downloadCrawlUrls = RequestHelper::getBoolFromRequest($request, '_download');
×
93
        $scheduledTime = $this->getScheduledTime(RequestHelper::getStringFromRequest($request, 'tstamp', 'now'));
×
94

95
        $this->incomingConfigurationSelection = RequestHelper::getArrayFromRequest($request, 'configurationSelection');
×
96

NEW
97
        $this->queueEntryWriter->setId = GeneralUtility::md5int(microtime());
×
98

99
        $queueRows = '';
×
100
        $noConfigurationSelected = $this->isNoConfigurationSelected();
×
101
        if ($noConfigurationSelected) {
×
102
            MessageUtility::addWarningMessage(
×
103
                $this->getLanguageService()->sL('crawler.messages:labels.noConfigSelected')
×
104
            );
×
105
        } else {
106
            if ($submitCrawlUrls) {
×
107
                $reason = new Reason();
×
108
                $reason->setReason(Reason::REASON_GUI_SUBMIT);
×
109
                $reason->setDetailText(
×
110
                    'The user ' . $GLOBALS['BE_USER']->user['username'] . ' added pages to the crawler queue manually'
×
111
                );
×
112
                $this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
×
113
            }
114

115
            $queueRows = $this->crawlerController->getPageTreeAndUrls(
×
116
                $this->pageUid,
×
117
                $crawlingDepth,
×
118
                $scheduledTime,
×
119
                $this->reqMinute,
×
120
                $submitCrawlUrls,
×
121
                $downloadCrawlUrls,
×
122
                // Do not filter any processing instructions
123
                [],
×
124
                $this->incomingConfigurationSelection
×
125
            );
×
126
        }
127

128
        // Download Urls to crawl:
129
        $downloadUrls = $this->crawlerController->downloadUrls;
×
130
        if ($downloadCrawlUrls) {
×
131
            // Creating output header:
132
            header('Content-Type: application/octet-stream');
×
133
            header('Content-Disposition: attachment; filename=CrawlerUrls.txt');
×
134

135
            // Printing the content of the CSV lines:
136
            echo implode(chr(self::CARRIAGE_RETURN) . chr(self::LINE_FEED), $downloadUrls);
×
137
            exit;
×
138
        }
139

140
        return $this->moduleTemplate->assignMultiple([
×
141
            'currentPageId' => $this->pageUid,
×
142
            'noConfigurationSelected' => $noConfigurationSelected,
×
143
            'submitCrawlUrls' => $submitCrawlUrls,
×
144
            'amountOfUrls' => count($this->crawlerController->duplicateTrack),
×
145
            'selectors' => $this->generateConfigurationSelectors($this->pageUid, (string) $crawlingDepth, $request),
×
146
            'queueRows' => $queueRows,
×
147
            'displayActions' => 0,
×
148
            'actionUrl' => $this->getActionUrl(),
×
149
            'logUrl' => $logUrl,
×
150
        ]);
×
151
    }
152

153
    /**
154
     * Generates the configuration selectors for compiling URLs:
155
     */
156
    private function generateConfigurationSelectors(
157
        int $pageId,
158
        string $crawlingDepth,
159
        ServerRequestInterface $request
160
    ): array {
161
        $selectors = [];
×
162
        $selectors['depth'] = $this->selectorBox(
×
163
            [
×
164
                0 => $this->getLanguageService()->sL('core.core:labels.depth_0'),
×
165
                1 => $this->getLanguageService()->sL('core.core:labels.depth_1'),
×
166
                2 => $this->getLanguageService()->sL('core.core:labels.depth_2'),
×
167
                3 => $this->getLanguageService()->sL('core.core:labels.depth_3'),
×
168
                4 => $this->getLanguageService()->sL('core.core:labels.depth_4'),
×
169
                99 => $this->getLanguageService()->sL('core.core:labels.depth_infi'),
×
170
            ],
×
171
            'crawlingDepth',
×
172
            $crawlingDepth,
×
173
            false
×
174
        );
×
175

176
        // Configurations
177
        $availableConfigurations = $this->pageConfigurationResolver->getConfigurationNamesForBranch(
×
178
            $pageId,
×
179
            (int) $crawlingDepth,
×
180
        );
×
181
        $selectors['configurations'] = $this->selectorBox(
×
182
            empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations),
×
183
            'configurationSelection',
×
184
            $this->incomingConfigurationSelection,
×
185
            true
×
186
        );
×
187

188
        // Scheduled time:
189
        $selectors['scheduled'] = $this->selectorBox(
×
190
            [
×
191
                'now' => $this->getLanguageService()->sL('crawler.messages:labels.time.now'),
×
192
                'midnight' => $this->getLanguageService()->sL('crawler.messages:labels.time.midnight'),
×
193
                '04:00' => $this->getLanguageService()->sL('crawler.messages:labels.time.4am'),
×
194
            ],
×
195
            'tstamp',
×
196
            RequestHelper::getStringFromRequest($request, 'tstamp', 'now'),
×
197
            false
×
198
        );
×
199

200
        return $selectors;
×
201
    }
202

203
    /**
204
     * Create selector box
205
     *
206
     * @param array $optArray Options key(value) => label pairs
207
     * @param string $name Selector box name
208
     * @param string|array|null $value Selector box value (array for multiple...)
209
     * @param boolean $multiple If set, will draw multiple box.
210
     *
211
     * @return string HTML select element
212
     */
213
    private function selectorBox($optArray, $name, string|array|null $value, bool $multiple): string
214
    {
215
        if (!is_string($value) && !is_array($value)) {
×
216
            $value = '';
×
217
        }
218

219
        $options = [];
×
220
        foreach ($optArray as $key => $val) {
×
221
            $selected = (
×
222
                !$multiple && is_string($value) && !strcmp($value, (string) $key)
×
223
            ) || (
×
224
                $multiple && in_array($key, (array) $value, true)
×
225
            );
×
226
            $options[] = '
×
227
                <option value="' . $key . '" ' . ($selected ? ' selected="selected"' : '') . '>' . htmlspecialchars(
×
228
                (string) $val,
×
229
                ENT_QUOTES | ENT_HTML5
×
230
            ) . '</option>';
×
231
        }
232

233
        return '<select class="form-select" name="' . htmlspecialchars(
×
234
            $name . ($multiple ? '[]' : ''),
×
235
            ENT_QUOTES | ENT_HTML5
×
236
        ) . '"' . ($multiple ? ' multiple' : '') . '>' . implode('', $options) . '</select>';
×
237
    }
238

239
    private function getScheduledTime(string $time): int
240
    {
241
        $scheduledTime = match ($time) {
×
242
            'midnight' => mktime(0, 0, 0),
×
243
            '04:00' => mktime(0, 0, 0) + 4 * 3600,
×
244
            default => time(),
×
245
        };
×
246

247
        if (!$scheduledTime) {
×
248
            return time();
×
249
        }
250

251
        return $scheduledTime;
×
252
    }
253

254
    /**
255
     * @throws RouteNotFoundException
256
     */
257
    private function getActionUrl(): UriInterface
258
    {
259
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
260
    }
261

262
    private function isNoConfigurationSelected(): bool
263
    {
264
        return empty($this->incomingConfigurationSelection)
×
265
            || (count($this->incomingConfigurationSelection) === 1 && empty($this->incomingConfigurationSelection[0]));
×
266
    }
267
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc