• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43583010903

14 Nov 2026 09:58AM UTC coverage: 67.631% (+0.1%) from 67.525%
43583010903

push

github

web-flow
!!! [TASK] Move the configuration lookup out of CrawlerController (#1334)

The new PageConfigurationResolver finds the crawler configurations of
pages, from page TSconfig and tx_crawler_configuration records. It
replaces CrawlerController::getUrlsForPageRow(), getUrlsForPageId(),
getPageTSconfigForId() and getConfigurationsForBranch(). The mount
point is a parameter instead of the CrawlerController::$MP state.

QueueService and the "Start Crawling" backend module use the resolver
directly. The getPageTSconfigForId and processUrls hooks receive the
resolver as parent object.

This lowers the cognitive complexity of CrawlerController from 74 to
54, and removes 2 entries from the Psalm baseline.

52 of 68 new or added lines in 4 files covered. (76.47%)

1 existing line in 1 file now uncovered.

1239 of 1832 relevant lines covered (67.63%)

5.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleStartCrawlingController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
24
use AOE\Crawler\Controller\CrawlerController;
25
use AOE\Crawler\Domain\Model\Reason;
26
use AOE\Crawler\Event\InvokeQueueChangeEvent;
27
use AOE\Crawler\Service\PageConfigurationResolver;
28
use AOE\Crawler\Utility\MessageUtility;
29
use Psr\Http\Message\ResponseInterface;
30
use Psr\Http\Message\ServerRequestInterface;
31
use Psr\Http\Message\UriInterface;
32
use Symfony\Contracts\Service\Attribute\Required;
33
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
34
use TYPO3\CMS\Backend\Routing\UriBuilder;
35
use TYPO3\CMS\Backend\Template\ModuleTemplate;
36
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
37
use TYPO3\CMS\Core\Utility\GeneralUtility;
38

39
/**
40
 * @internal since v12.0.0
41
 */
42
class BackendModuleStartCrawlingController extends AbstractBackendModuleController implements BackendModuleControllerInterface
43
{
44
    private const BACKEND_MODULE = 'web_site_crawler_start';
45
    private const LINE_FEED = 10;
46
    private const CARRIAGE_RETURN = 13;
47
    private int $reqMinute = 1000;
48
    private EventDispatcher $eventDispatcher;
49

50
    /**
51
     * @var array holds the selection of configuration from the configuration selector box
52
     */
53
    private $incomingConfigurationSelection = [];
54

55
    public function __construct(
56
        private readonly CrawlerController $crawlerController,
57
        private readonly PageConfigurationResolver $pageConfigurationResolver,
58
        private readonly UriBuilder $backendUriBuilder,
59
    ) {
60
    }
×
61

62
    #[Required]
63
    public function setEventDispatcher(): void
64
    {
65
        $this->eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);
×
66
    }
67

68
    #[\Override]
69
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
70
    {
71
        $this->checkPhpBinaryForDirectRequests($this->extensionSettings);
×
72
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
73
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
74
        $this->moduleTemplate->makeDocHeaderModuleMenu([
×
75
            'id' => $request->getQueryParams()['id'] ?? -1,
×
76
        ]);
×
77

78
        $this->assignValues($request);
×
79
        return $this->moduleTemplate->renderResponse('Backend/ShowCrawlerInformation');
×
80
    }
81

82
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
83
    {
84
        $logUrl = $this->backendUriBuilder->buildUriFromRoute('web_site_crawler_log', [
×
85
            'id' => $this->pageUid,
×
86
        ]);
×
87

88
        $crawlingDepth = RequestHelper::getIntFromRequest($request, 'crawlingDepth');
×
89
        $submitCrawlUrls = RequestHelper::getBoolFromRequest($request, '_crawl');
×
90
        $downloadCrawlUrls = RequestHelper::getBoolFromRequest($request, '_download');
×
91
        $scheduledTime = $this->getScheduledTime(RequestHelper::getStringFromRequest($request, 'tstamp', 'now'));
×
92

93
        $this->incomingConfigurationSelection = RequestHelper::getArrayFromRequest($request, 'configurationSelection');
×
94

95
        //$this->crawlerController = $this->getCrawlerController();
96
        $this->crawlerController->setID = GeneralUtility::md5int(microtime());
×
97

98
        $queueRows = '';
×
99
        $noConfigurationSelected = $this->isNoConfigurationSelected();
×
100
        if ($noConfigurationSelected) {
×
101
            MessageUtility::addWarningMessage(
×
102
                $this->getLanguageService()->sL('crawler.messages:labels.noConfigSelected')
×
103
            );
×
104
        } else {
105
            if ($submitCrawlUrls) {
×
106
                $reason = new Reason();
×
107
                $reason->setReason(Reason::REASON_GUI_SUBMIT);
×
108
                $reason->setDetailText(
×
109
                    'The user ' . $GLOBALS['BE_USER']->user['username'] . ' added pages to the crawler queue manually'
×
110
                );
×
111
                $this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
×
112
            }
113

114
            $queueRows = $this->crawlerController->getPageTreeAndUrls(
×
115
                $this->pageUid,
×
116
                $crawlingDepth,
×
117
                $scheduledTime,
×
118
                $this->reqMinute,
×
119
                $submitCrawlUrls,
×
120
                $downloadCrawlUrls,
×
121
                // Do not filter any processing instructions
122
                [],
×
123
                $this->incomingConfigurationSelection
×
124
            );
×
125
        }
126

127
        // Download Urls to crawl:
128
        $downloadUrls = $this->crawlerController->downloadUrls;
×
129
        if ($downloadCrawlUrls) {
×
130
            // Creating output header:
131
            header('Content-Type: application/octet-stream');
×
132
            header('Content-Disposition: attachment; filename=CrawlerUrls.txt');
×
133

134
            // Printing the content of the CSV lines:
135
            echo implode(chr(self::CARRIAGE_RETURN) . chr(self::LINE_FEED), $downloadUrls);
×
136
            exit;
×
137
        }
138

139
        return $this->moduleTemplate->assignMultiple([
×
140
            'currentPageId' => $this->pageUid,
×
141
            'noConfigurationSelected' => $noConfigurationSelected,
×
142
            'submitCrawlUrls' => $submitCrawlUrls,
×
143
            'amountOfUrls' => count($this->crawlerController->duplicateTrack),
×
144
            'selectors' => $this->generateConfigurationSelectors($this->pageUid, (string) $crawlingDepth, $request),
×
145
            'queueRows' => $queueRows,
×
146
            'displayActions' => 0,
×
147
            'actionUrl' => $this->getActionUrl(),
×
148
            'logUrl' => $logUrl,
×
149
        ]);
×
150
    }
151

152
    /**
153
     * Generates the configuration selectors for compiling URLs:
154
     */
155
    private function generateConfigurationSelectors(
156
        int $pageId,
157
        string $crawlingDepth,
158
        ServerRequestInterface $request
159
    ): array {
160
        $selectors = [];
×
161
        $selectors['depth'] = $this->selectorBox(
×
162
            [
×
163
                0 => $this->getLanguageService()->sL('core.core:labels.depth_0'),
×
164
                1 => $this->getLanguageService()->sL('core.core:labels.depth_1'),
×
165
                2 => $this->getLanguageService()->sL('core.core:labels.depth_2'),
×
166
                3 => $this->getLanguageService()->sL('core.core:labels.depth_3'),
×
167
                4 => $this->getLanguageService()->sL('core.core:labels.depth_4'),
×
168
                99 => $this->getLanguageService()->sL('core.core:labels.depth_infi'),
×
169
            ],
×
170
            'crawlingDepth',
×
171
            $crawlingDepth,
×
172
            false
×
173
        );
×
174

175
        // Configurations
NEW
176
        $availableConfigurations = $this->pageConfigurationResolver->getConfigurationNamesForBranch(
×
177
            $pageId,
×
178
            (int) $crawlingDepth,
×
179
        );
×
180
        $selectors['configurations'] = $this->selectorBox(
×
181
            empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations),
×
182
            'configurationSelection',
×
183
            $this->incomingConfigurationSelection,
×
184
            true
×
185
        );
×
186

187
        // Scheduled time:
188
        $selectors['scheduled'] = $this->selectorBox(
×
189
            [
×
190
                'now' => $this->getLanguageService()->sL('crawler.messages:labels.time.now'),
×
191
                'midnight' => $this->getLanguageService()->sL('crawler.messages:labels.time.midnight'),
×
192
                '04:00' => $this->getLanguageService()->sL('crawler.messages:labels.time.4am'),
×
193
            ],
×
194
            'tstamp',
×
195
            RequestHelper::getStringFromRequest($request, 'tstamp', 'now'),
×
196
            false
×
197
        );
×
198

199
        return $selectors;
×
200
    }
201

202
    /**
203
     * Create selector box
204
     *
205
     * @param array $optArray Options key(value) => label pairs
206
     * @param string $name Selector box name
207
     * @param string|array|null $value Selector box value (array for multiple...)
208
     * @param boolean $multiple If set, will draw multiple box.
209
     *
210
     * @return string HTML select element
211
     */
212
    private function selectorBox($optArray, $name, string|array|null $value, bool $multiple): string
213
    {
214
        if (!is_string($value) && !is_array($value)) {
×
215
            $value = '';
×
216
        }
217

218
        $options = [];
×
219
        foreach ($optArray as $key => $val) {
×
220
            $selected = (
×
221
                !$multiple && is_string($value) && !strcmp($value, (string) $key)
×
222
            ) || (
×
223
                $multiple && in_array($key, (array) $value, true)
×
224
            );
×
225
            $options[] = '
×
226
                <option value="' . $key . '" ' . ($selected ? ' selected="selected"' : '') . '>' . htmlspecialchars(
×
227
                (string) $val,
×
228
                ENT_QUOTES | ENT_HTML5
×
229
            ) . '</option>';
×
230
        }
231

232
        return '<select class="form-select" name="' . htmlspecialchars(
×
233
            $name . ($multiple ? '[]' : ''),
×
234
            ENT_QUOTES | ENT_HTML5
×
235
        ) . '"' . ($multiple ? ' multiple' : '') . '>' . implode('', $options) . '</select>';
×
236
    }
237

238
    private function getScheduledTime(string $time): int
239
    {
240
        $scheduledTime = match ($time) {
×
241
            'midnight' => mktime(0, 0, 0),
×
242
            '04:00' => mktime(0, 0, 0) + 4 * 3600,
×
243
            default => time(),
×
244
        };
×
245

246
        if (!$scheduledTime) {
×
247
            return time();
×
248
        }
249

250
        return $scheduledTime;
×
251
    }
252

253
    /**
254
     * @throws RouteNotFoundException
255
     */
256
    private function getActionUrl(): UriInterface
257
    {
258
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
259
    }
260

261
    private function isNoConfigurationSelected(): bool
262
    {
263
        return empty($this->incomingConfigurationSelection)
×
264
            || (count($this->incomingConfigurationSelection) === 1 && empty($this->incomingConfigurationSelection[0]));
×
265
    }
266
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc