• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

tomasnorre / crawler / 43501585105

13 Nov 2026 08:26PM UTC coverage: 67.056% (+0.07%) from 66.983%
43501585105

push

github

web-flow
[BUGFIX] Cache exclude page trees, split ConfigurationService to reduce complexity (#1328)

120 of 177 new or added lines in 4 files covered. (67.8%)

1262 of 1882 relevant lines covered (67.06%)

5.09 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.46
/Classes/Service/ConfigurationService.php
1
<?php
2

3
declare(strict_types=1);
4

5
namespace AOE\Crawler\Service;
6

7
/*
8
 * (c) 2022 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9
 *
10
 * This file is part of the TYPO3 Crawler Extension.
11
 *
12
 * It is free software; you can redistribute it and/or modify it under
13
 * the terms of the GNU General Public License, either version 2
14
 * of the License, or any later version.
15
 *
16
 * For the full copyright and license information, please read the
17
 * LICENSE.txt file that was distributed with this source code.
18
 *
19
 * The TYPO3 project - inspiring people to share!
20
 */
21

22
use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
23
use AOE\Crawler\Domain\Repository\ConfigurationRepository;
24
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
25
use TYPO3\CMS\Core\Authentication\GroupResolver;
26
use TYPO3\CMS\Core\Core\Bootstrap;
27
use TYPO3\CMS\Core\EventDispatcher\NoopEventDispatcher;
28
use TYPO3\CMS\Core\TypoScript\AST\AstBuilder;
29
use TYPO3\CMS\Core\TypoScript\TypoScriptStringFactory;
30
use TYPO3\CMS\Core\Utility\GeneralUtility;
31
use TYPO3\CMS\Core\Utility\MathUtility;
32

33
/**
34
 * @internal since v9.2.5
35
 */
36
class ConfigurationService
37
{
38
    private const DEFAULT_MAX_URLS_TO_COMPILE = 10_000;
39

40
    private readonly array $extensionSettings;
41

42
    public function __construct(
43
        private readonly UrlService $urlService,
44
        private readonly ConfigurationRepository $configurationRepository,
45
        private readonly GroupResolver $groupResolver,
46
        private readonly ParameterExpander $parameterExpander,
47
        private readonly ExcludeStringExpander $excludeStringExpander,
48
    ) {
49
        $this->extensionSettings = GeneralUtility::makeInstance(
54 ✔
50
            ExtensionConfigurationProvider::class
54 ✔
51
        )->getExtensionConfiguration();
54 ✔
52
    }
53

54
    public static function removeDisallowedConfigurations(array $allowedConfigurations, array $configurations): array
55
    {
56
        if (empty($allowedConfigurations)) {
14 ✔
57
            return $configurations;
5 ✔
58
        }
59
        //         remove configuration that does not match the current selection
60
        foreach ($configurations as $confKey => $confArray) {
9 ✔
61
            if (!in_array($confKey, $allowedConfigurations, true)) {
9 ✔
62
                unset($configurations[$confKey]);
8 ✔
63
            }
64
        }
65

66
        return $configurations;
9 ✔
67
    }
68

69
    public function getConfigurationFromPageTS(
70
        array $pageTSConfig,
71
        int $pageId,
72
        array $res,
73
        string $mountPoint = ''
74
    ): array {
75
        $crawlerCfg = $pageTSConfig['tx_crawler.']['crawlerCfg.']['paramSets.'] ?? [];
43 ✔
76
        foreach ($crawlerCfg as $key => $values) {
43 ✔
77
            if (!is_array($values)) {
35 ✔
78
                continue;
35 ✔
79
            }
80
            $key = str_replace('.', '', (string) $key);
34 ✔
81
            // Sub configuration for a single configuration string:
82
            $subCfg = (array) $crawlerCfg[$key . '.'];
34 ✔
83
            $subCfg['key'] = $key;
34 ✔
84

85
            if (strcmp($subCfg['procInstrFilter'] ?? '', '')) {
34 ✔
86
                $subCfg['procInstrFilter'] = implode(',', GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']));
9 ✔
87
            }
88

89
            if (!$this->appliesToPage((string) ($subCfg['pidsOnly'] ?? ''), $pageId)) {
34 ✔
90
                continue;
1 ✔
91
            }
92

93
            $url = '?id=' . $pageId;
33 ✔
94
            $url .= $mountPoint !== '' ? '&MP=' . $mountPoint : '';
33 ✔
95
            $res[$key] = $this->buildConfiguration($subCfg, (string) $crawlerCfg[$key], $pageId, $url, 'pagets');
33 ✔
96
        }
97
        return $res;
43 ✔
98
    }
99

100
    public function getConfigurationFromDatabase(int $pageId, array $res): array
101
    {
102
        $crawlerConfigurations = $this->configurationRepository->getCrawlerConfigurationRecordsFromRootLine($pageId);
11 ✔
103
        foreach ($crawlerConfigurations as $configurationRecord) {
11 ✔
104
            $key = $configurationRecord['name'];
8 ✔
105

106
            // Previously defined paramSets are not overwritten
107
            if (isset($res[$key])
8 ✔
108
                || !$this->hasAccess($configurationRecord)
8 ✔
109
                || !$this->appliesToPage((string) $configurationRecord['pidsonly'], $pageId)
8 ✔
110
                || in_array($pageId, $this->expandExcludeString((string) $configurationRecord['exclude']), true)
8 ✔
111
            ) {
NEW
112
                continue;
×
113
            }
114

115
            $res[$key] = $this->buildConfiguration(
8 ✔
116
                $this->buildSubConfigurationFromRecord($configurationRecord),
8 ✔
117
                (string) $configurationRecord['configuration'],
8 ✔
118
                $pageId,
8 ✔
119
                '?id=' . $pageId,
8 ✔
120
                'tx_crawler_configuration_' . $configurationRecord['uid']
8 ✔
121
            );
8 ✔
122
        }
123
        return $res;
11 ✔
124
    }
125

126
    /**
127
     * @return array<int, int>
128
     */
129
    public function expandExcludeString(string $excludeString): array
130
    {
131
        return $this->excludeStringExpander->expand($excludeString);
11 ✔
132
    }
133

134
    /**
135
     * A configuration applies to all pages, unless "pidsOnly" limits it to some pages
136
     */
137
    private function appliesToPage(string $pidsOnly, int $pageId): bool
138
    {
139
        if ($pidsOnly === '') {
42 ✔
140
            return true;
33 ✔
141
        }
142
        return in_array((string) $pageId, GeneralUtility::trimExplode(',', $pidsOnly, true), true);
9 ✔
143
    }
144

145
    private function hasAccess(array $configurationRecord): bool
146
    {
147
        if (empty($configurationRecord['begroups']) || $this->getBackendUser()->isAdmin()) {
8 ✔
148
            return true;
8 ✔
149
        }
NEW
150
        return UserService::hasGroupAccess(
×
NEW
151
            implode(',', $this->groupResolver->resolveGroupsForUser([$this->getBackendUser()], 'be_groups')),
×
NEW
152
            $configurationRecord['begroups']
×
NEW
153
        );
×
154
    }
155

156
    private function buildSubConfigurationFromRecord(array $configurationRecord): array
157
    {
158
        $typoScriptTree = GeneralUtility::makeInstance(TypoScriptStringFactory::class)->parseFromString(
8 ✔
159
            $configurationRecord['processing_instruction_parameters_ts'],
8 ✔
160
            new AstBuilder(new NoopEventDispatcher())
8 ✔
161
        );
8 ✔
162

163
        return [
8 ✔
164
            'procInstrFilter' => $configurationRecord['processing_instruction_filter'],
8 ✔
165
            'procInstrParams.' => $typoScriptTree->toArray(),
8 ✔
166
            'baseUrl' => $configurationRecord['base_url'],
8 ✔
167
            'force_ssl' => (int) $configurationRecord['force_ssl'],
8 ✔
168
            'userGroups' => $configurationRecord['fegroups'],
8 ✔
169
            'exclude' => $configurationRecord['exclude'],
8 ✔
170
            'key' => $configurationRecord['name'],
8 ✔
171
        ];
8 ✔
172
    }
173

174
    private function buildConfiguration(
175
        array $subCfg,
176
        string $parameters,
177
        int $pageId,
178
        string $baseUrl,
179
        string $origin
180
    ): array {
181
        $configuration = [];
41 ✔
182
        $configuration['subCfg'] = $subCfg;
41 ✔
183
        $configuration['paramParsed'] = GeneralUtility::explodeUrl2Array($parameters);
41 ✔
184
        $configuration['paramExpanded'] = $this->parameterExpander->expand($configuration['paramParsed'], $pageId);
41 ✔
185
        $configuration['origin'] = $origin;
41 ✔
186
        $configuration['URLs'] = $this->urlService->compileUrls(
41 ✔
187
            $configuration['paramExpanded'],
41 ✔
188
            [$baseUrl],
41 ✔
189
            $this->getMaxUrlsToCompile()
41 ✔
190
        );
41 ✔
191
        return $configuration;
41 ✔
192
    }
193

194
    private function getMaxUrlsToCompile(): int
195
    {
196
        return MathUtility::forceIntegerInRange(
41 ✔
197
            $this->extensionSettings['maxCompileUrls'] ?? self::DEFAULT_MAX_URLS_TO_COMPILE,
41 ✔
198
            1,
41 ✔
199
            1_000_000_000,
41 ✔
200
            self::DEFAULT_MAX_URLS_TO_COMPILE
41 ✔
201
        );
41 ✔
202
    }
203

204
    private function getBackendUser(): BackendUserAuthentication
205
    {
206
        // Make sure the _cli_ user is loaded
NEW
207
        Bootstrap::initializeBackendAuthentication();
×
NEW
208
        return $GLOBALS['BE_USER'];
×
209
    }
210
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc