• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43449756443

13 Nov 2026 03:09PM UTC coverage: 67.299% (-4.0%) from 71.322%
43449756443

push

github

web-flow
!!! [FEATURE] Process the crawler queue with the TYPO3 message bus (#1322)

The crawler queue is no longer processed by the extension's own process
management. Each queue entry is dispatched as a CrawlUrlMessage to the
TYPO3 message bus (symfony/messenger) and crawled by CrawlUrlHandler in
a worker started with "typo3 messenger:consume".

Queue processing:
- Queue entries are still stored in tx_crawler_queue. The message only
  carries the queue id and is dispatched when the entry is added from the
  backend module, the crawler:buildQueue command or on page save.
- Scheduled entries are delayed with a DelayStamp.
- The Before/AfterQueueItemAddedEvent are dispatched by the handler.
- Re-crawling from the log module dispatches a forced message.

Priority crawling:
- Three Doctrine transports crawler_high, crawler and crawler_low.
- The transport is chosen by the sitemap_priority of the page (EXT:seo),
  re-crawls from the log module use the high priority transport.
- Run the worker with "messenger:consume crawler_high crawler crawler_low"
  to crawl high priority pages first.
- If the routing of CrawlUrlMessage is changed, e.g. to the synchronous
  "default" transport, the routing is respected.

Breaking changes:
- The crawler:processQueue command, the process backend module, the
  process status AJAX route and the table tx_crawler_process are removed,
  together with the process related extension settings.
- Removed classes: Crawler, ProcessService, ProcessRepository, the
  Process models, cleaners and manager, the Sleeper helpers, HookUtility,
  BackendModuleLinkService, AfterQueueItemAddedEventListener,
  Domain\Model\Configuration, Domain\Model\Queue, Value\CrawlAction,
  CrawlerObjectException and TimeStampException.
- Removed methods: CrawlerController::readUrl(), readUrlFromArray() and
  the deprecated drawURLs_PIfilter(), as well as the process related
  methods of QueueRepository.
- The extension requires symfony/messenger and psr/event-dispatc... (continued)

74 of 91 new or added lines in 10 files covered. (81.32%)

6 existing lines in 1 file now uncovered.

1278 of 1899 relevant lines covered (67.3%)

4.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/Classes/Controller/Backend/AbstractBackendModuleController.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Controller\Backend;
6

7
/*
8
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
23
use AOE\Crawler\Utility\MessageUtility;
24
use AOE\Crawler\Utility\PhpBinaryUtility;
25
use Psr\Http\Message\ServerRequestInterface;
26
use Symfony\Contracts\Service\Attribute\Required;
27
use TYPO3\CMS\Backend\Template\ModuleTemplate;
28
use TYPO3\CMS\Backend\Template\ModuleTemplateFactory;
29
use TYPO3\CMS\Backend\Utility\BackendUtility;
30
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
31
use TYPO3\CMS\Core\Localization\LanguageService;
32
use TYPO3\CMS\Core\Type\Bitmask\Permission;
33
use TYPO3\CMS\Core\Utility\CommandUtility;
34
use TYPO3\CMS\Core\Utility\GeneralUtility;
35

36
/**
37
 * @internal since v12.0.0
38
 */
39
abstract class AbstractBackendModuleController
40
{
41
    protected int $pageUid;
42
    protected ModuleTemplate $moduleTemplate;
43
    protected array $extensionSettings = [];
44

45
    #[Required]
46
    public function setExtensionSettings(): void
47
    {
48
        $configurationProvider = GeneralUtility::makeInstance(ExtensionConfigurationProvider::class);
×
49
        $this->extensionSettings = $configurationProvider->getExtensionConfiguration();
×
50
    }
51

52
    protected function setupView(ServerRequestInterface $request, int $pageUid): ModuleTemplate
53
    {
54
        $moduleTemplate = (GeneralUtility::makeInstance(ModuleTemplateFactory::class))->create($request);
×
55
        $permissionClause = $this->getBackendUserAuthentication()->getPagePermsClause(Permission::PAGE_SHOW);
×
56
        $pageRecord = BackendUtility::readPageAccess($pageUid, $permissionClause);
×
57
        if ($pageRecord) {
×
58
            $moduleTemplate->getDocHeaderComponent()->setPageBreadcrumb($pageRecord);
×
59
        }
60

61
        return $moduleTemplate;
×
62
    }
63

64
    /**
65
     * Direct requests (makeDirectRequests) render pages in a PHP sub-process, which requires a working PHP binary.
66
     */
67
    protected function checkPhpBinaryForDirectRequests(array $extensionSettings): void
68
    {
NEW
69
        if (!($extensionSettings['makeDirectRequests'] ?? false)) {
×
NEW
70
            return;
×
71
        }
72

73
        $exitCode = 0;
×
74
        $out = [];
×
75
        CommandUtility::exec(PhpBinaryUtility::getPhpBinary() . ' -v', $out, $exitCode);
×
76
        if ($exitCode > 0) {
×
77
            MessageUtility::addErrorMessage(
×
78
                sprintf($this->getLanguageService()->sL(
×
79
                    'crawler.messages:message.phpBinaryNotFound'
×
80
                ), htmlspecialchars((string) $extensionSettings['phpPath'], ENT_QUOTES | ENT_HTML5))
×
81
            );
×
82
        }
83
    }
84

85
    protected function getModuleMenu(): array
86
    {
87
        return [
×
88
            'logDepth' => [
×
89
                0 => $this->getLanguageService()->sL('core.core:labels.depth_0'),
×
90
                1 => $this->getLanguageService()->sL('core.core:labels.depth_1'),
×
91
                2 => $this->getLanguageService()->sL('core.core:labels.depth_2'),
×
92
                3 => $this->getLanguageService()->sL('core.core:labels.depth_3'),
×
93
                4 => $this->getLanguageService()->sL('core.core:labels.depth_4'),
×
94
                99 => $this->getLanguageService()->sL('core.core:labels.depth_infi'),
×
95
            ],
×
96
            'displayLog' => [
×
97
                'all' => $this->getLanguageService()->sL('crawler.messages:labels.all'),
×
98
                'pending' => $this->getLanguageService()->sL('crawler.messages:labels.pending'),
×
99
                'finished' => $this->getLanguageService()->sL('crawler.messages:labels.finished'),
×
100
            ],
×
101
            'itemsPerPage' => [
×
102
                '5' => $this->getLanguageService()->sL('crawler.messages:labels.itemsPerPage.5'),
×
103
                '10' => $this->getLanguageService()->sL('crawler.messages:labels.itemsPerPage.10'),
×
104
                '50' => $this->getLanguageService()->sL('crawler.messages:labels.itemsPerPage.50'),
×
105
                '0' => $this->getLanguageService()->sL('crawler.messages:labels.itemsPerPage.0'),
×
106
            ],
×
107
        ];
×
108
    }
109

110
    protected function getLanguageService(): LanguageService
111
    {
112
        return $GLOBALS['LANG'];
×
113
    }
114

115
    protected function getBackendUserAuthentication(): BackendUserAuthentication
116
    {
117
        return $GLOBALS['BE_USER'];
×
118
    }
119
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc