• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43818818671

16 Nov 2026 06:44AM UTC coverage: 68.855% (-0.07%) from 68.921%
43818818671

push

github

web-flow
!!! [TASK] Keep the state of compiling URLs in a CrawlRun (#1340)

* !!! [TASK] Keep the state of compiling URLs in a CrawlRun

QueueEntryWriter and PageTreeUrlCollector are shared services, but kept
the options and results of a run in public properties: the set id, the
handled URLs, the download URLs, the URL list and the collected queue
entries. Callers had to set and reset them, and state could leak from
one run into the next.

AOE\Crawler\Queue\CrawlRun now holds the options of one run and collects
its results. It is created per run and handed over as parameter, so both
services are stateless.

* QueueEntryWriter::compileUrlList() and addUrl() take a CrawlRun
* PageTreeUrlCollector::collect() and getQueueRowsForPage() take a CrawlRun
* QueueEntryWriter::addCallback() takes the set id as optional parameter
* BackendModuleStartCrawlingController no longer needs the QueueEntryWriter

The new unit tests of compileUrlList() covered code that had no unit tests
before, and the mutants escaping there lowered the covered code MSI.

* CrawlRun builds the key of a handled URL and calculates the scheduled
  time of the next URL, both tested exactly
* compileUrlList() checks the processing instruction filter once instead
  of for every URL, it does not change within a configuration
* Tests for base URL and force_ssl, URLs that cannot be built, user groups
  and processing instruction filters of handled URLs, numeric
  configuration keys and several parsed parameters

63 of 76 new or added lines in 6 files covered. (82.89%)

1269 of 1843 relevant lines covered (68.86%)

5.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleStartCrawlingController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
24
use AOE\Crawler\Domain\Model\Reason;
25
use AOE\Crawler\Event\InvokeQueueChangeEvent;
26
use AOE\Crawler\Queue\CrawlRun;
27
use AOE\Crawler\Service\PageConfigurationResolver;
28
use AOE\Crawler\Service\PageTreeUrlCollector;
29
use AOE\Crawler\Utility\MessageUtility;
30
use Psr\Http\Message\ResponseInterface;
31
use Psr\Http\Message\ServerRequestInterface;
32
use Psr\Http\Message\UriInterface;
33
use Symfony\Contracts\Service\Attribute\Required;
34
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
35
use TYPO3\CMS\Backend\Routing\UriBuilder;
36
use TYPO3\CMS\Backend\Template\ModuleTemplate;
37
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
38
use TYPO3\CMS\Core\Utility\GeneralUtility;
39

40
/**
41
 * @internal since v12.0.0
42
 */
43
class BackendModuleStartCrawlingController extends AbstractBackendModuleController implements BackendModuleControllerInterface
44
{
45
    private const BACKEND_MODULE = 'web_site_crawler_start';
46
    private const LINE_FEED = 10;
47
    private const CARRIAGE_RETURN = 13;
48
    private int $reqMinute = 1000;
49
    private EventDispatcher $eventDispatcher;
50

51
    /**
52
     * @var array holds the selection of configuration from the configuration selector box
53
     */
54
    private $incomingConfigurationSelection = [];
55

56
    public function __construct(
57
        private readonly PageTreeUrlCollector $pageTreeUrlCollector,
58
        private readonly PageConfigurationResolver $pageConfigurationResolver,
59
        private readonly UriBuilder $backendUriBuilder,
60
    ) {
61
    }
×
62

63
    #[Required]
64
    public function setEventDispatcher(): void
65
    {
66
        $this->eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);
×
67
    }
68

69
    #[\Override]
70
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
71
    {
72
        $this->checkPhpBinaryForDirectRequests($this->extensionSettings);
×
73
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
74
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
75
        $this->moduleTemplate->makeDocHeaderModuleMenu([
×
76
            'id' => $request->getQueryParams()['id'] ?? -1,
×
77
        ]);
×
78

79
        $this->assignValues($request);
×
80
        return $this->moduleTemplate->renderResponse('Backend/ShowCrawlerInformation');
×
81
    }
82

83
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
84
    {
85
        $logUrl = $this->backendUriBuilder->buildUriFromRoute('web_site_crawler_log', [
×
86
            'id' => $this->pageUid,
×
87
        ]);
×
88

89
        $crawlingDepth = RequestHelper::getIntFromRequest($request, 'crawlingDepth');
×
90
        $submitCrawlUrls = RequestHelper::getBoolFromRequest($request, '_crawl');
×
91
        $downloadCrawlUrls = RequestHelper::getBoolFromRequest($request, '_download');
×
92
        $scheduledTime = $this->getScheduledTime(RequestHelper::getStringFromRequest($request, 'tstamp', 'now'));
×
93

94
        $this->incomingConfigurationSelection = RequestHelper::getArrayFromRequest($request, 'configurationSelection');
×
95

NEW
96
        $run = new CrawlRun(
×
NEW
97
            $scheduledTime,
×
NEW
98
            $this->reqMinute,
×
NEW
99
            submitUrls: $submitCrawlUrls,
×
NEW
100
            collectDownloadUrls: $downloadCrawlUrls,
×
NEW
101
            configurationSelection: $this->incomingConfigurationSelection,
×
NEW
102
        );
×
103

104
        $queueRows = '';
×
105
        $noConfigurationSelected = $this->isNoConfigurationSelected();
×
106
        if ($noConfigurationSelected) {
×
107
            MessageUtility::addWarningMessage(
×
108
                $this->getLanguageService()->sL('crawler.messages:labels.noConfigSelected')
×
109
            );
×
110
        } else {
111
            if ($submitCrawlUrls) {
×
112
                $reason = new Reason();
×
113
                $reason->setReason(Reason::REASON_GUI_SUBMIT);
×
114
                $reason->setDetailText(
×
115
                    'The user ' . $GLOBALS['BE_USER']->user['username'] . ' added pages to the crawler queue manually'
×
116
                );
×
117
                $this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
×
118
            }
119

NEW
120
            $queueRows = $this->pageTreeUrlCollector->collect($this->pageUid, $crawlingDepth, $run);
×
121
        }
122

123
        // Download Urls to crawl:
NEW
124
        $downloadUrls = $run->getDownloadUrls();
×
125
        if ($downloadCrawlUrls) {
×
126
            // Creating output header:
127
            header('Content-Type: application/octet-stream');
×
128
            header('Content-Disposition: attachment; filename=CrawlerUrls.txt');
×
129

130
            // Printing the content of the CSV lines:
131
            echo implode(chr(self::CARRIAGE_RETURN) . chr(self::LINE_FEED), $downloadUrls);
×
132
            exit;
×
133
        }
134

135
        return $this->moduleTemplate->assignMultiple([
×
136
            'currentPageId' => $this->pageUid,
×
137
            'noConfigurationSelected' => $noConfigurationSelected,
×
138
            'submitCrawlUrls' => $submitCrawlUrls,
×
NEW
139
            'amountOfUrls' => $run->countHandledUrls(),
×
140
            'selectors' => $this->generateConfigurationSelectors($this->pageUid, (string) $crawlingDepth, $request),
×
141
            'queueRows' => $queueRows,
×
142
            'displayActions' => 0,
×
143
            'actionUrl' => $this->getActionUrl(),
×
144
            'logUrl' => $logUrl,
×
145
        ]);
×
146
    }
147

148
    /**
149
     * Generates the configuration selectors for compiling URLs:
150
     */
151
    private function generateConfigurationSelectors(
152
        int $pageId,
153
        string $crawlingDepth,
154
        ServerRequestInterface $request
155
    ): array {
156
        $selectors = [];
×
157
        $selectors['depth'] = $this->selectorBox(
×
158
            [
×
159
                0 => $this->getLanguageService()->sL('core.core:labels.depth_0'),
×
160
                1 => $this->getLanguageService()->sL('core.core:labels.depth_1'),
×
161
                2 => $this->getLanguageService()->sL('core.core:labels.depth_2'),
×
162
                3 => $this->getLanguageService()->sL('core.core:labels.depth_3'),
×
163
                4 => $this->getLanguageService()->sL('core.core:labels.depth_4'),
×
164
                99 => $this->getLanguageService()->sL('core.core:labels.depth_infi'),
×
165
            ],
×
166
            'crawlingDepth',
×
167
            $crawlingDepth,
×
168
            false
×
169
        );
×
170

171
        // Configurations
172
        $availableConfigurations = $this->pageConfigurationResolver->getConfigurationNamesForBranch(
×
173
            $pageId,
×
174
            (int) $crawlingDepth,
×
175
        );
×
176
        $selectors['configurations'] = $this->selectorBox(
×
177
            empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations),
×
178
            'configurationSelection',
×
179
            $this->incomingConfigurationSelection,
×
180
            true
×
181
        );
×
182

183
        // Scheduled time:
184
        $selectors['scheduled'] = $this->selectorBox(
×
185
            [
×
186
                'now' => $this->getLanguageService()->sL('crawler.messages:labels.time.now'),
×
187
                'midnight' => $this->getLanguageService()->sL('crawler.messages:labels.time.midnight'),
×
188
                '04:00' => $this->getLanguageService()->sL('crawler.messages:labels.time.4am'),
×
189
            ],
×
190
            'tstamp',
×
191
            RequestHelper::getStringFromRequest($request, 'tstamp', 'now'),
×
192
            false
×
193
        );
×
194

195
        return $selectors;
×
196
    }
197

198
    /**
199
     * Create selector box
200
     *
201
     * @param array $optArray Options key(value) => label pairs
202
     * @param string $name Selector box name
203
     * @param string|array|null $value Selector box value (array for multiple...)
204
     * @param boolean $multiple If set, will draw multiple box.
205
     *
206
     * @return string HTML select element
207
     */
208
    private function selectorBox($optArray, $name, string|array|null $value, bool $multiple): string
209
    {
210
        if (!is_string($value) && !is_array($value)) {
×
211
            $value = '';
×
212
        }
213

214
        $options = [];
×
215
        foreach ($optArray as $key => $val) {
×
216
            $selected = (
×
217
                !$multiple && is_string($value) && !strcmp($value, (string) $key)
×
218
            ) || (
×
219
                $multiple && in_array($key, (array) $value, true)
×
220
            );
×
221
            $options[] = '
×
222
                <option value="' . $key . '" ' . ($selected ? ' selected="selected"' : '') . '>' . htmlspecialchars(
×
223
                (string) $val,
×
224
                ENT_QUOTES | ENT_HTML5
×
225
            ) . '</option>';
×
226
        }
227

228
        return '<select class="form-select" name="' . htmlspecialchars(
×
229
            $name . ($multiple ? '[]' : ''),
×
230
            ENT_QUOTES | ENT_HTML5
×
231
        ) . '"' . ($multiple ? ' multiple' : '') . '>' . implode('', $options) . '</select>';
×
232
    }
233

234
    private function getScheduledTime(string $time): int
235
    {
236
        $scheduledTime = match ($time) {
×
237
            'midnight' => mktime(0, 0, 0),
×
238
            '04:00' => mktime(0, 0, 0) + 4 * 3600,
×
239
            default => time(),
×
240
        };
×
241

242
        if (!$scheduledTime) {
×
243
            return time();
×
244
        }
245

246
        return $scheduledTime;
×
247
    }
248

249
    /**
250
     * @throws RouteNotFoundException
251
     */
252
    private function getActionUrl(): UriInterface
253
    {
254
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
255
    }
256

257
    private function isNoConfigurationSelected(): bool
258
    {
259
        return empty($this->incomingConfigurationSelection)
×
260
            || (count($this->incomingConfigurationSelection) === 1 && empty($this->incomingConfigurationSelection[0]));
×
261
    }
262
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc