• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43601825635

14 Nov 2026 01:51PM UTC coverage: 68.804% (-0.02%) from 68.824%
43601825635

push

github

web-flow
!!! [TASK] Replace CrawlerController with PageTreeUrlCollector (#1338)

The page tree walking moves to AOE\Crawler\Service\PageTreeUrlCollector,
which removes the last code of CrawlerController.

* getPageTreeAndUrls() is now collect(), with native types
* drawURLs_addRowsForPage() is now getQueueRowsForPage() and takes the
  mount point as parameter instead of the public property $MP
* The PHPStan ignore rule for public methods in controllers is removed,
  it matched nothing anymore

7 of 15 new or added lines in 3 files covered. (46.67%)

1277 of 1856 relevant lines covered (68.8%)

5.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/BackendModuleStartCrawlingController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Controller\Backend\Helper\RequestHelper;
23
use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
24
use AOE\Crawler\Domain\Model\Reason;
25
use AOE\Crawler\Event\InvokeQueueChangeEvent;
26
use AOE\Crawler\Queue\QueueEntryWriter;
27
use AOE\Crawler\Service\PageConfigurationResolver;
28
use AOE\Crawler\Service\PageTreeUrlCollector;
29
use AOE\Crawler\Utility\MessageUtility;
30
use Psr\Http\Message\ResponseInterface;
31
use Psr\Http\Message\ServerRequestInterface;
32
use Psr\Http\Message\UriInterface;
33
use Symfony\Contracts\Service\Attribute\Required;
34
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
35
use TYPO3\CMS\Backend\Routing\UriBuilder;
36
use TYPO3\CMS\Backend\Template\ModuleTemplate;
37
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
38
use TYPO3\CMS\Core\Utility\GeneralUtility;
39

40
/**
41
 * @internal since v12.0.0
42
 */
43
class BackendModuleStartCrawlingController extends AbstractBackendModuleController implements BackendModuleControllerInterface
44
{
45
    private const BACKEND_MODULE = 'web_site_crawler_start';
46
    private const LINE_FEED = 10;
47
    private const CARRIAGE_RETURN = 13;
48
    private int $reqMinute = 1000;
49
    private EventDispatcher $eventDispatcher;
50

51
    /**
52
     * @var array holds the selection of configuration from the configuration selector box
53
     */
54
    private $incomingConfigurationSelection = [];
55

56
    public function __construct(
57
        private readonly PageTreeUrlCollector $pageTreeUrlCollector,
58
        private readonly PageConfigurationResolver $pageConfigurationResolver,
59
        private readonly QueueEntryWriter $queueEntryWriter,
60
        private readonly UriBuilder $backendUriBuilder,
61
    ) {
62
    }
×
63

64
    #[Required]
65
    public function setEventDispatcher(): void
66
    {
67
        $this->eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);
×
68
    }
69

70
    #[\Override]
71
    public function handleRequest(ServerRequestInterface $request): ResponseInterface
72
    {
73
        $this->checkPhpBinaryForDirectRequests($this->extensionSettings);
×
74
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
×
75
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
×
76
        $this->moduleTemplate->makeDocHeaderModuleMenu([
×
77
            'id' => $request->getQueryParams()['id'] ?? -1,
×
78
        ]);
×
79

80
        $this->assignValues($request);
×
81
        return $this->moduleTemplate->renderResponse('Backend/ShowCrawlerInformation');
×
82
    }
83

84
    private function assignValues(ServerRequestInterface $request): ModuleTemplate
85
    {
86
        $logUrl = $this->backendUriBuilder->buildUriFromRoute('web_site_crawler_log', [
×
87
            'id' => $this->pageUid,
×
88
        ]);
×
89

90
        $crawlingDepth = RequestHelper::getIntFromRequest($request, 'crawlingDepth');
×
91
        $submitCrawlUrls = RequestHelper::getBoolFromRequest($request, '_crawl');
×
92
        $downloadCrawlUrls = RequestHelper::getBoolFromRequest($request, '_download');
×
93
        $scheduledTime = $this->getScheduledTime(RequestHelper::getStringFromRequest($request, 'tstamp', 'now'));
×
94

95
        $this->incomingConfigurationSelection = RequestHelper::getArrayFromRequest($request, 'configurationSelection');
×
96

97
        $this->queueEntryWriter->setId = GeneralUtility::md5int(microtime());
×
98

99
        $queueRows = '';
×
100
        $noConfigurationSelected = $this->isNoConfigurationSelected();
×
101
        if ($noConfigurationSelected) {
×
102
            MessageUtility::addWarningMessage(
×
103
                $this->getLanguageService()->sL('crawler.messages:labels.noConfigSelected')
×
104
            );
×
105
        } else {
106
            if ($submitCrawlUrls) {
×
107
                $reason = new Reason();
×
108
                $reason->setReason(Reason::REASON_GUI_SUBMIT);
×
109
                $reason->setDetailText(
×
110
                    'The user ' . $GLOBALS['BE_USER']->user['username'] . ' added pages to the crawler queue manually'
×
111
                );
×
112
                $this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
×
113
            }
114

NEW
115
            $queueRows = $this->pageTreeUrlCollector->collect(
×
116
                $this->pageUid,
×
117
                $crawlingDepth,
×
118
                $scheduledTime,
×
119
                $this->reqMinute,
×
120
                $submitCrawlUrls,
×
121
                $downloadCrawlUrls,
×
122
                // Do not filter any processing instructions
123
                [],
×
124
                $this->incomingConfigurationSelection
×
125
            );
×
126
        }
127

128
        // Download Urls to crawl:
NEW
129
        $downloadUrls = $this->pageTreeUrlCollector->downloadUrls;
×
130
        if ($downloadCrawlUrls) {
×
131
            // Creating output header:
132
            header('Content-Type: application/octet-stream');
×
133
            header('Content-Disposition: attachment; filename=CrawlerUrls.txt');
×
134

135
            // Printing the content of the CSV lines:
136
            echo implode(chr(self::CARRIAGE_RETURN) . chr(self::LINE_FEED), $downloadUrls);
×
137
            exit;
×
138
        }
139

140
        return $this->moduleTemplate->assignMultiple([
×
141
            'currentPageId' => $this->pageUid,
×
142
            'noConfigurationSelected' => $noConfigurationSelected,
×
143
            'submitCrawlUrls' => $submitCrawlUrls,
×
NEW
144
            'amountOfUrls' => count($this->pageTreeUrlCollector->duplicateTrack),
×
145
            'selectors' => $this->generateConfigurationSelectors($this->pageUid, (string) $crawlingDepth, $request),
×
146
            'queueRows' => $queueRows,
×
147
            'displayActions' => 0,
×
148
            'actionUrl' => $this->getActionUrl(),
×
149
            'logUrl' => $logUrl,
×
150
        ]);
×
151
    }
152

153
    /**
154
     * Generates the configuration selectors for compiling URLs:
155
     */
156
    private function generateConfigurationSelectors(
157
        int $pageId,
158
        string $crawlingDepth,
159
        ServerRequestInterface $request
160
    ): array {
161
        $selectors = [];
×
162
        $selectors['depth'] = $this->selectorBox(
×
163
            [
×
164
                0 => $this->getLanguageService()->sL('core.core:labels.depth_0'),
×
165
                1 => $this->getLanguageService()->sL('core.core:labels.depth_1'),
×
166
                2 => $this->getLanguageService()->sL('core.core:labels.depth_2'),
×
167
                3 => $this->getLanguageService()->sL('core.core:labels.depth_3'),
×
168
                4 => $this->getLanguageService()->sL('core.core:labels.depth_4'),
×
169
                99 => $this->getLanguageService()->sL('core.core:labels.depth_infi'),
×
170
            ],
×
171
            'crawlingDepth',
×
172
            $crawlingDepth,
×
173
            false
×
174
        );
×
175

176
        // Configurations
177
        $availableConfigurations = $this->pageConfigurationResolver->getConfigurationNamesForBranch(
×
178
            $pageId,
×
179
            (int) $crawlingDepth,
×
180
        );
×
181
        $selectors['configurations'] = $this->selectorBox(
×
182
            empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations),
×
183
            'configurationSelection',
×
184
            $this->incomingConfigurationSelection,
×
185
            true
×
186
        );
×
187

188
        // Scheduled time:
189
        $selectors['scheduled'] = $this->selectorBox(
×
190
            [
×
191
                'now' => $this->getLanguageService()->sL('crawler.messages:labels.time.now'),
×
192
                'midnight' => $this->getLanguageService()->sL('crawler.messages:labels.time.midnight'),
×
193
                '04:00' => $this->getLanguageService()->sL('crawler.messages:labels.time.4am'),
×
194
            ],
×
195
            'tstamp',
×
196
            RequestHelper::getStringFromRequest($request, 'tstamp', 'now'),
×
197
            false
×
198
        );
×
199

200
        return $selectors;
×
201
    }
202

203
    /**
204
     * Create selector box
205
     *
206
     * @param array $optArray Options key(value) => label pairs
207
     * @param string $name Selector box name
208
     * @param string|array|null $value Selector box value (array for multiple...)
209
     * @param boolean $multiple If set, will draw multiple box.
210
     *
211
     * @return string HTML select element
212
     */
213
    private function selectorBox($optArray, $name, string|array|null $value, bool $multiple): string
214
    {
215
        if (!is_string($value) && !is_array($value)) {
×
216
            $value = '';
×
217
        }
218

219
        $options = [];
×
220
        foreach ($optArray as $key => $val) {
×
221
            $selected = (
×
222
                !$multiple && is_string($value) && !strcmp($value, (string) $key)
×
223
            ) || (
×
224
                $multiple && in_array($key, (array) $value, true)
×
225
            );
×
226
            $options[] = '
×
227
                <option value="' . $key . '" ' . ($selected ? ' selected="selected"' : '') . '>' . htmlspecialchars(
×
228
                (string) $val,
×
229
                ENT_QUOTES | ENT_HTML5
×
230
            ) . '</option>';
×
231
        }
232

233
        return '<select class="form-select" name="' . htmlspecialchars(
×
234
            $name . ($multiple ? '[]' : ''),
×
235
            ENT_QUOTES | ENT_HTML5
×
236
        ) . '"' . ($multiple ? ' multiple' : '') . '>' . implode('', $options) . '</select>';
×
237
    }
238

239
    private function getScheduledTime(string $time): int
240
    {
241
        $scheduledTime = match ($time) {
×
242
            'midnight' => mktime(0, 0, 0),
×
243
            '04:00' => mktime(0, 0, 0) + 4 * 3600,
×
244
            default => time(),
×
245
        };
×
246

247
        if (!$scheduledTime) {
×
248
            return time();
×
249
        }
250

251
        return $scheduledTime;
×
252
    }
253

254
    /**
255
     * @throws RouteNotFoundException
256
     */
257
    private function getActionUrl(): UriInterface
258
    {
259
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
×
260
    }
261

262
    private function isNoConfigurationSelected(): bool
263
    {
264
        return empty($this->incomingConfigurationSelection)
×
265
            || (count($this->incomingConfigurationSelection) === 1 && empty($this->incomingConfigurationSelection[0]));
×
266
    }
267
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc