17555324436

Committed 08 Sep 2025 03:07PM UTC coverage: 68.691% (-0.02%) from 68.713%

Build # 17555324436

Build Type

Pull #1174

github

Committed by

web-flow

Commit Message

Merge 123205f7f into b044d2640

Pull Request Pull Request #1174: [TASK] Refactor ConfigurationServer->expandParameters()

Coverage Stats

39 of 40 new or added lines in 1 file covered. (97.5%)

1878 of 2734 relevant lines covered (68.69%)

3.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.76

/Classes/Service/ConfigurationService.php

<?php

declare(strict_types=1);

namespace AOE\Crawler\Service;

/*
 * (c) 2022 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
 *
 * This file is part of the TYPO3 Crawler Extension.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
use AOE\Crawler\Domain\Repository\ConfigurationRepository;
use Doctrine\DBAL\ArrayParameterType;
use TYPO3\CMS\Backend\Tree\View\PageTreeView;
use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
use TYPO3\CMS\Core\Core\Bootstrap;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
use TYPO3\CMS\Core\Domain\Repository\PageRepository;
use TYPO3\CMS\Core\EventDispatcher\NoopEventDispatcher;
use TYPO3\CMS\Core\Type\Bitmask\Permission;
use TYPO3\CMS\Core\TypoScript\AST\AstBuilder;
use TYPO3\CMS\Core\TypoScript\TypoScriptStringFactory;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;

/**
 * @internal since v9.2.5
 */
class ConfigurationService
{
    /**
     * @var BackendUserAuthentication|null
     */
    private $backendUser;
    private readonly array $extensionSettings;

    public function __construct(
        private readonly UrlService $urlService,
        private readonly ConfigurationRepository $configurationRepository
    ) {
        $this->extensionSettings = GeneralUtility::makeInstance(
            ExtensionConfigurationProvider::class
        )->getExtensionConfiguration();
    }

    public static function removeDisallowedConfigurations(array $allowedConfigurations, array $configurations): array
    {
        if (empty($allowedConfigurations)) {
            return $configurations;
        }
        //         remove configuration that does not match the current selection
        foreach ($configurations as $confKey => $confArray) {
            if (!in_array($confKey, $allowedConfigurations, true)) {
                unset($configurations[$confKey]);
            }
        }

        return $configurations;
    }

    public function getConfigurationFromPageTS(
        array $pageTSConfig,
        int $pageId,
        array $res,
        string $mountPoint = ''
    ): array {
        $defaultCompileUrls = 10_000;
        $maxUrlsToCompile = MathUtility::forceIntegerInRange(
            $this->extensionSettings['maxCompileUrls'] ?? $defaultCompileUrls,
            1,
            1_000_000_000,
            $defaultCompileUrls
        );
        $crawlerCfg = $pageTSConfig['tx_crawler.']['crawlerCfg.']['paramSets.'] ?? [];
        foreach ($crawlerCfg as $key => $values) {
            if (!is_array($values)) {
                continue;
            }
            $key = str_replace('.', '', (string) $key);
            // Sub configuration for a single configuration string:
            $subCfg = (array) $crawlerCfg[$key . '.'];
            $subCfg['key'] = $key;

            if (strcmp($subCfg['procInstrFilter'] ?? '', '')) {
                $subCfg['procInstrFilter'] = implode(',', GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']));
            }
            $pidOnlyList = implode(',', GeneralUtility::trimExplode(',', $subCfg['pidsOnly'] ?? '', true));

            // process configuration if it is not page-specific or if the specific page is the current page:
            // TODO: Check if $pidOnlyList can be kept as Array instead of imploded
            if (!strcmp((string) ($subCfg['pidsOnly'] ?? ''), '') || GeneralUtility::inList(
                $pidOnlyList,
                strval($pageId)
            )) {
                // Explode, process etc.:
                $res[$key] = [];
                $res[$key]['subCfg'] = $subCfg;
                $res[$key]['paramParsed'] = GeneralUtility::explodeUrl2Array($crawlerCfg[$key]);
                $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $pageId);
                $res[$key]['origin'] = 'pagets';

                $url = '?id=' . $pageId;
                $url .= $mountPoint !== '' ? '&MP=' . $mountPoint : '';
                $res[$key]['URLs'] = $this->urlService->compileUrls(
                    $res[$key]['paramExpanded'],
                    [$url],
                    $maxUrlsToCompile
                );
            }
        }
        return $res;
    }

    public function getConfigurationFromDatabase(int $pageId, array $res): array
    {
        $maxUrlsToCompile = MathUtility::forceIntegerInRange(
            $this->extensionSettings['maxCompileUrls'],
            1,
            1_000_000_000,
            10000
        );

        $crawlerConfigurations = $this->configurationRepository->getCrawlerConfigurationRecordsFromRootLine($pageId);
        foreach ($crawlerConfigurations as $configurationRecord) {
            // check access to the configuration record
            if (empty($configurationRecord['begroups']) || $this->getBackendUser()->isAdmin() || UserService::hasGroupAccess(
                $this->getBackendUser()->user['usergroup_cached_list'],
                $configurationRecord['begroups']
            )) {
                $pidOnlyList = implode(',', GeneralUtility::trimExplode(',', $configurationRecord['pidsonly'], true));

                // process configuration if it is not page-specific or if the specific page is the current page:
                // TODO: Check if $pidOnlyList can be kept as Array instead of imploded
                if (!strcmp((string) $configurationRecord['pidsonly'], '') || GeneralUtility::inList(
                    $pidOnlyList,
                    strval($pageId)
                )) {
                    $key = $configurationRecord['name'];

                    // don't overwrite previously defined paramSets
                    if (!isset($res[$key])) {
                        /** @var TypoScriptStringFactory $typoScriptStringFactory */
                        $typoScriptStringFactory = GeneralUtility::makeInstance(TypoScriptStringFactory::class);
                        $typoScriptTree = $typoScriptStringFactory->parseFromString(
                            $configurationRecord['processing_instruction_parameters_ts'],
                            new AstBuilder(new NoopEventDispatcher())
                        );

                        $subCfg = [
                            'procInstrFilter' => $configurationRecord['processing_instruction_filter'],
                            'procInstrParams.' => $typoScriptTree->toArray(),
                            'baseUrl' => $configurationRecord['base_url'],
                            'force_ssl' => (int) $configurationRecord['force_ssl'],
                            'userGroups' => $configurationRecord['fegroups'],
                            'exclude' => $configurationRecord['exclude'],
                            'key' => $key,
                        ];

                        if (!in_array($pageId, $this->expandExcludeString($subCfg['exclude']), true)) {
                            $res[$key] = [];
                            $res[$key]['subCfg'] = $subCfg;
                            $res[$key]['paramParsed'] = GeneralUtility::explodeUrl2Array(
                                $configurationRecord['configuration']
                            );
                            $res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $pageId);
                            $res[$key]['URLs'] = $this->urlService->compileUrls(
                                $res[$key]['paramExpanded'],
                                ['?id=' . $pageId],
                                $maxUrlsToCompile
                            );
                            $res[$key]['origin'] = 'tx_crawler_configuration_' . $configurationRecord['uid'];
                        }
                    }
                }
            }
        }
        return $res;
    }

    public function expandExcludeString(string $excludeString): array
    {
        // internal static caches;
        static $expandedExcludeStringCache;
        static $treeCache = [];

        if (!empty($expandedExcludeStringCache[$excludeString])) {
            return $expandedExcludeStringCache[$excludeString];
        }

        $pidList = [];

        if (!empty($excludeString)) {
            /** @var PageTreeView $tree */
            $tree = GeneralUtility::makeInstance(PageTreeView::class);
            $tree->init('AND ' . $this->getBackendUser()->getPagePermsClause(Permission::PAGE_SHOW));

            $excludeParts = GeneralUtility::trimExplode(',', $excludeString);

            foreach ($excludeParts as $excludePart) {
                $explodedExcludePart = GeneralUtility::trimExplode('+', $excludePart);
                $pid = isset($explodedExcludePart[0]) ? (int) $explodedExcludePart[0] : 0;
                $depth = isset($explodedExcludePart[1]) ? (int) $explodedExcludePart[1] : null;

                // default is "page only" = "depth=0"
                if (empty($depth)) {
                    $depth = (str_contains($excludePart, '+')) ? 99 : 0;
                }

                $pidList[] = $pid;
                if ($depth > 0) {
                    $pidList = $this->expandPidList($treeCache, $pid, $depth, $tree, $pidList);
                }
            }
        }

        $expandedExcludeStringCache[$excludeString] = array_unique($pidList);

        return $expandedExcludeStringCache[$excludeString];
    }

    /**
     * Expands the parameters configuration into individual values.
     *
     * Syntax of values:
     * - Literal values are taken as-is unless wrapped in [ ].
     * - If wrapped in [ ]:
     *   - Parts are separated by "|" and expanded individually.
     *   - Supported parts:
     *       - "x-y"       → Integer range (inclusive, max 1000 values)
     *       - "_TABLE:..." → Lookup values from a TCA table
     *       - Otherwise   → Literal value
     */
    private function expandParameters(array $paramArray, int $pid): array
    {
        /** @var array<string, string|int|float|null> $paramArray */
        foreach ($paramArray as $parameter => $rawValue) {
            $value = trim((string) $rawValue);

            if ($this->isWrappedInSquareBrackets($value)) {
                /** @var array<int, string|int> $expanded */
                $expanded = $this->expandBracketedValue($value, $pid, $parameter, $paramArray);
                $paramArray[$parameter] = $expanded;
            } else {
                /** @var array<int, string> $paramArray */
                $paramArray[$parameter] = [$value];
            }
        }

        /** @var array<string, array<int, string|int>> $paramArray */
        return $paramArray;
    }

    /**
     * Expands a value wrapped in [ ] into an array of individual values.
     *
     * @param array<string, mixed>             $paramArray
     * @return array<int, string|int>
     */
    private function expandBracketedValue(string $value, int $pid, string $parameter, array $paramArray): array
    {
        $expandedValues = [];
        $innerValue = substr($value, 1, -1);
        $parts = explode('|', $innerValue);

        foreach ($parts as $part) {
            $part = trim($part);

            if ($this->isIntegerRange($part)) {
                $expandedValues = array_merge($expandedValues, $this->expandIntegerRange($part));
            } elseif (str_starts_with($part, '_TABLE:')) {
                $expandedValues = array_merge(
                    $expandedValues,
                    $this->expandTableLookup($part, $pid, $parameter, $paramArray)
                );
            } else {
                $expandedValues[] = $part;
            }

            // Allow custom hooks to modify the expanded values
            $paramArray = $this->runExpandParametersHook($paramArray, $parameter, $part, $pid);
        }

        return array_values(array_unique($expandedValues));
    }

    /**
     * Checks whether a part is an integer range like "1-34" or "-40--30".
     */
    private function isIntegerRange(string $part): bool
    {
        return (bool) preg_match('/^(-?\d+)\s*-\s*(-?\d+)$/', $part);
    }

    /**
     * Expands an integer range string into an array of integers.
     *
     * @return array<int, int>
     */
    private function expandIntegerRange(string $part): array
    {
        preg_match('/^(-?\d+)\s*-\s*(-?\d+)$/', $part, $matches);

        [, $start, $end] = $matches;
        $start = (int) $start;
        $end = (int) $end;

        if ($start > $end) {
            [$start, $end] = [$end, $start];
        }

        $range = range($start, $end);

        // Safety limit to prevent runaway ranges
        return count($range) > 1000 ? array_slice($range, 0, 1000) : $range;
    }

    /**
     * Expands a _TABLE lookup definition into values from the TCA table.
     *
     * @param array<string, mixed>             $paramArray
     * @return array<int, string|int>
     */
    private function expandTableLookup(string $definition, int $pid, string $parameter, array $paramArray): array
    {
        $subparts = GeneralUtility::trimExplode(';', $definition);
        $options = [];

        foreach ($subparts as $subpart) {
            [$key, $value] = GeneralUtility::trimExplode(':', $subpart);
            $options[$key] = $value;
        }

        if (!isset($GLOBALS['TCA'][$options['_TABLE']])) {
            return []; // invalid table
        }

        return $this->extractParamsFromCustomTable($options, $pid, $paramArray, $parameter);
    }

    private function isWrappedInSquareBrackets(string $string): bool
    {
        return str_starts_with($string, '[') && str_ends_with($string, ']');
    }

    /**
     * @return BackendUserAuthentication
     */
    private function getBackendUser()
    {
        // Make sure the _cli_ user is loaded
        Bootstrap::initializeBackendAuthentication();
        if ($this->backendUser === null) {
            $this->backendUser = $GLOBALS['BE_USER'];
        }
        return $this->backendUser;
    }

    private function getQueryBuilder(string $table): QueryBuilder
    {
        return GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
    }

    /**
     * @psalm-param array-key $parameter
     */
    private function runExpandParametersHook(array $paramArray, int|string $parameter, string $path, int $pid): array
    {
        if (is_array(
            $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] ?? null
        )) {
            $_params = [
                'pObj' => &$this,
                'paramArray' => &$paramArray,
                'currentKey' => $parameter,
                'currentValue' => $path,
                'pid' => $pid,
            ];
            foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] as $_funcRef) {
                GeneralUtility::callUserFunction($_funcRef, $_params, $this);
            }
        }
        return $paramArray;
    }

    private function getPidArray(int $recursiveDepth, int $lookUpPid): array
    {
        if ($recursiveDepth > 0) {
            $pageRepository = GeneralUtility::makeInstance(PageRepository::class);
            $pidArray = $pageRepository->getPageIdsRecursive([$lookUpPid], $recursiveDepth);
        } else {
            $pidArray = [$lookUpPid];
        }
        return $pidArray;
    }

    private function expandPidList(array $treeCache, int $pid, int $depth, PageTreeView $tree, array $pidList): array
    {
        if (empty($treeCache[$pid][$depth])) {
            $tree->reset();
            $tree->getTree($pid, $depth);
            $treeCache[$pid][$depth] = $tree->tree;
        }

        foreach ($treeCache[$pid][$depth] as $data) {
            $pidList[] = (int) $data['row']['uid'];
        }
        return $pidList;
    }

    private function extractParamsFromCustomTable(
        array $subpartParams,
        int $pid,
        array $paramArray,
        int|string $parameter
    ): array {
        $lookUpPid = isset($subpartParams['_PID']) ? (int) $subpartParams['_PID'] : $pid;
        $recursiveDepth = isset($subpartParams['_RECURSIVE']) ? (int) $subpartParams['_RECURSIVE'] : 0;
        $pidField = isset($subpartParams['_PIDFIELD']) ? trim((string) $subpartParams['_PIDFIELD']) : 'pid';
        $where = $subpartParams['_WHERE'] ?? '';
        $addTable = $subpartParams['_ADDTABLE'] ?? '';

        $fieldName = ($subpartParams['_FIELD'] ?? '') ?: 'uid';
        if ($fieldName === 'uid' || $GLOBALS['TCA'][$subpartParams['_TABLE']]['columns'][$fieldName]) {
            $queryBuilder = $this->getQueryBuilder($subpartParams['_TABLE']);
            $pidArray = $this->getPidArray($recursiveDepth, $lookUpPid);

            $queryBuilder->getRestrictions()
                ->removeAll()
                ->add(GeneralUtility::makeInstance(DeletedRestriction::class));

            $queryBuilder
                ->select($fieldName)
                ->from($subpartParams['_TABLE'])
                ->where(
                    $queryBuilder->expr()->in(
                        $pidField,
                        $queryBuilder->createNamedParameter($pidArray, ArrayParameterType::INTEGER)
                    ),
                    $where
                );

            if (!empty($addTable)) {
                // TODO: Check if this works as intended!
                $addTables = GeneralUtility::trimExplode(',', $addTable, true);
                foreach ($addTables as $table) {
                    $queryBuilder->from($table);
                }
            }
            $transOrigPointerField = $GLOBALS['TCA'][$subpartParams['_TABLE']]['ctrl']['transOrigPointerField'] ?? false;

            if (($subpartParams['_ENABLELANG'] ?? false) && $transOrigPointerField) {
                $queryBuilder->andWhere($queryBuilder->expr()->lte($transOrigPointerField, 0));
            }

            $statement = $queryBuilder->executeQuery();

            $rows = [];
            while ($row = $statement->fetchAssociative()) {
                $rows[$row[$fieldName]] = $row;
            }

            if (is_array($rows)) {
                $paramArray[$parameter] = array_merge($paramArray[$parameter], array_keys($rows));
            }
        }
        return $paramArray;
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace AOE\Crawler\Service;
6
7	/*
8	* (c) 2022 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9	*
10	* This file is part of the TYPO3 Crawler Extension.
11	*
12	* It is free software; you can redistribute it and/or modify it under
13	* the terms of the GNU General Public License, either version 2
14	* of the License, or any later version.
15	*
16	* For the full copyright and license information, please read the
17	* LICENSE.txt file that was distributed with this source code.
18	*
19	* The TYPO3 project - inspiring people to share!
20	*/
21
22	use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
23	use AOE\Crawler\Domain\Repository\ConfigurationRepository;
24	use Doctrine\DBAL\ArrayParameterType;
25	use TYPO3\CMS\Backend\Tree\View\PageTreeView;
26	use TYPO3\CMS\Core\Authentication\BackendUserAuthentication;
27	use TYPO3\CMS\Core\Core\Bootstrap;
28	use TYPO3\CMS\Core\Database\ConnectionPool;
29	use TYPO3\CMS\Core\Database\Query\QueryBuilder;
30	use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
31	use TYPO3\CMS\Core\Domain\Repository\PageRepository;
32	use TYPO3\CMS\Core\EventDispatcher\NoopEventDispatcher;
33	use TYPO3\CMS\Core\Type\Bitmask\Permission;
34	use TYPO3\CMS\Core\TypoScript\AST\AstBuilder;
35	use TYPO3\CMS\Core\TypoScript\TypoScriptStringFactory;
36	use TYPO3\CMS\Core\Utility\GeneralUtility;
37	use TYPO3\CMS\Core\Utility\MathUtility;
38
39	/**
40	* @internal since v9.2.5
41	*/
42	class ConfigurationService
43	{
44	/**
45	* @var BackendUserAuthentication\|null
46	*/
47	private $backendUser;
48	private readonly array $extensionSettings;
49
50	public function __construct(
51	private readonly UrlService $urlService,
52	private readonly ConfigurationRepository $configurationRepository
53	) {
54	$this->extensionSettings = GeneralUtility::makeInstance(	24✔
55	ExtensionConfigurationProvider::class	24✔
56	)->getExtensionConfiguration();	24✔
57	}
58
59	public static function removeDisallowedConfigurations(array $allowedConfigurations, array $configurations): array
60	{
61	if (empty($allowedConfigurations)) {	11✔
62	return $configurations;	5✔
63	}
64	// remove configuration that does not match the current selection
65	foreach ($configurations as $confKey => $confArray) {	6✔
66	if (!in_array($confKey, $allowedConfigurations, true)) {	6✔
67	unset($configurations[$confKey]);	6✔
68	}
69	}
70
71	return $configurations;	6✔
72	}
73
74	public function getConfigurationFromPageTS(
75	array $pageTSConfig,
76	int $pageId,
77	array $res,
78	string $mountPoint = ''
79	): array {
80	$defaultCompileUrls = 10_000;	15✔
81	$maxUrlsToCompile = MathUtility::forceIntegerInRange(	15✔
82	$this->extensionSettings['maxCompileUrls'] ?? $defaultCompileUrls,	15✔
83	1,	15✔
84	1_000_000_000,	15✔
85	$defaultCompileUrls	15✔
86	);	15✔
87	$crawlerCfg = $pageTSConfig['tx_crawler.']['crawlerCfg.']['paramSets.'] ?? [];	15✔
88	foreach ($crawlerCfg as $key => $values) {	15✔
89	if (!is_array($values)) {	9✔
90	continue;	9✔
91	}
92	$key = str_replace('.', '', (string) $key);	9✔
93	// Sub configuration for a single configuration string:
94	$subCfg = (array) $crawlerCfg[$key . '.'];	9✔
95	$subCfg['key'] = $key;	9✔
96
97	if (strcmp($subCfg['procInstrFilter'] ?? '', '')) {	9✔
98	$subCfg['procInstrFilter'] = implode(',', GeneralUtility::trimExplode(',', $subCfg['procInstrFilter']));	9✔
99	}
100	$pidOnlyList = implode(',', GeneralUtility::trimExplode(',', $subCfg['pidsOnly'] ?? '', true));	9✔
101
102	// process configuration if it is not page-specific or if the specific page is the current page:
103	// TODO: Check if $pidOnlyList can be kept as Array instead of imploded
104	if (!strcmp((string) ($subCfg['pidsOnly'] ?? ''), '') \|\| GeneralUtility::inList(	9✔
105	$pidOnlyList,	9✔
106	strval($pageId)	9✔
107	)) {	9✔
108	// Explode, process etc.:
109	$res[$key] = [];	9✔
110	$res[$key]['subCfg'] = $subCfg;	9✔
111	$res[$key]['paramParsed'] = GeneralUtility::explodeUrl2Array($crawlerCfg[$key]);	9✔
112	$res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $pageId);	9✔
113	$res[$key]['origin'] = 'pagets';	9✔
114
115	$url = '?id=' . $pageId;	9✔
116	$url .= $mountPoint !== '' ? '&MP=' . $mountPoint : '';	9✔
117	$res[$key]['URLs'] = $this->urlService->compileUrls(	9✔
118	$res[$key]['paramExpanded'],	9✔
119	[$url],	9✔
120	$maxUrlsToCompile	9✔
121	);	9✔
122	}
123	}
124	return $res;	15✔
125	}
126
127	public function getConfigurationFromDatabase(int $pageId, array $res): array
128	{
129	$maxUrlsToCompile = MathUtility::forceIntegerInRange(	9✔
130	$this->extensionSettings['maxCompileUrls'],	9✔
131	1,	9✔
132	1_000_000_000,	9✔
133	10000	9✔
134	);	9✔
135
136	$crawlerConfigurations = $this->configurationRepository->getCrawlerConfigurationRecordsFromRootLine($pageId);	9✔
137	foreach ($crawlerConfigurations as $configurationRecord) {	9✔
138	// check access to the configuration record
139	if (empty($configurationRecord['begroups']) \|\| $this->getBackendUser()->isAdmin() \|\| UserService::hasGroupAccess(	6✔
140	$this->getBackendUser()->user['usergroup_cached_list'],	6✔
141	$configurationRecord['begroups']	6✔
142	)) {	6✔
143	$pidOnlyList = implode(',', GeneralUtility::trimExplode(',', $configurationRecord['pidsonly'], true));	6✔
144
145	// process configuration if it is not page-specific or if the specific page is the current page:
146	// TODO: Check if $pidOnlyList can be kept as Array instead of imploded
147	if (!strcmp((string) $configurationRecord['pidsonly'], '') \|\| GeneralUtility::inList(	6✔
148	$pidOnlyList,	6✔
149	strval($pageId)	6✔
150	)) {	6✔
151	$key = $configurationRecord['name'];	6✔
152
153	// don't overwrite previously defined paramSets
154	if (!isset($res[$key])) {	6✔
155	/** @var TypoScriptStringFactory $typoScriptStringFactory */
156	$typoScriptStringFactory = GeneralUtility::makeInstance(TypoScriptStringFactory::class);	6✔
157	$typoScriptTree = $typoScriptStringFactory->parseFromString(	6✔
158	$configurationRecord['processing_instruction_parameters_ts'],	6✔
159	new AstBuilder(new NoopEventDispatcher())	6✔
160	);	6✔
161
162	$subCfg = [	6✔
163	'procInstrFilter' => $configurationRecord['processing_instruction_filter'],	6✔
164	'procInstrParams.' => $typoScriptTree->toArray(),	6✔
165	'baseUrl' => $configurationRecord['base_url'],	6✔
166	'force_ssl' => (int) $configurationRecord['force_ssl'],	6✔
167	'userGroups' => $configurationRecord['fegroups'],	6✔
168	'exclude' => $configurationRecord['exclude'],	6✔
169	'key' => $key,	6✔
170	];	6✔
171
172	if (!in_array($pageId, $this->expandExcludeString($subCfg['exclude']), true)) {	6✔
173	$res[$key] = [];	6✔
174	$res[$key]['subCfg'] = $subCfg;	6✔
175	$res[$key]['paramParsed'] = GeneralUtility::explodeUrl2Array(	6✔
176	$configurationRecord['configuration']	6✔
177	);	6✔
178	$res[$key]['paramExpanded'] = $this->expandParameters($res[$key]['paramParsed'], $pageId);	6✔
179	$res[$key]['URLs'] = $this->urlService->compileUrls(	6✔
180	$res[$key]['paramExpanded'],	6✔
181	['?id=' . $pageId],	6✔
182	$maxUrlsToCompile	6✔
183	);	6✔
184	$res[$key]['origin'] = 'tx_crawler_configuration_' . $configurationRecord['uid'];	6✔
185	}
186	}
187	}
188	}
189	}
190	return $res;	9✔
191	}
192
193	public function expandExcludeString(string $excludeString): array
194	{
195	// internal static caches;
196	static $expandedExcludeStringCache;	7✔
197	static $treeCache = [];	7✔
198
199	if (!empty($expandedExcludeStringCache[$excludeString])) {	7✔
200	return $expandedExcludeStringCache[$excludeString];	×
201	}
202
203	$pidList = [];	7✔
204
205	if (!empty($excludeString)) {	7✔
206	/** @var PageTreeView $tree */
207	$tree = GeneralUtility::makeInstance(PageTreeView::class);	1✔
208	$tree->init('AND ' . $this->getBackendUser()->getPagePermsClause(Permission::PAGE_SHOW));	1✔
209
210	$excludeParts = GeneralUtility::trimExplode(',', $excludeString);	1✔
211
212	foreach ($excludeParts as $excludePart) {	1✔
213	$explodedExcludePart = GeneralUtility::trimExplode('+', $excludePart);	1✔
214	$pid = isset($explodedExcludePart[0]) ? (int) $explodedExcludePart[0] : 0;	1✔
215	$depth = isset($explodedExcludePart[1]) ? (int) $explodedExcludePart[1] : null;	1✔
216
217	// default is "page only" = "depth=0"
218	if (empty($depth)) {	1✔
219	$depth = (str_contains($excludePart, '+')) ? 99 : 0;	1✔
220	}
221
222	$pidList[] = $pid;	1✔
223	if ($depth > 0) {	1✔
224	$pidList = $this->expandPidList($treeCache, $pid, $depth, $tree, $pidList);	×
225	}
226	}
227	}
228
229	$expandedExcludeStringCache[$excludeString] = array_unique($pidList);	7✔
230
231	return $expandedExcludeStringCache[$excludeString];	7✔
232	}
233
234	/**
235	* Expands the parameters configuration into individual values.
236	*
237	* Syntax of values:
238	* - Literal values are taken as-is unless wrapped in [ ].
239	* - If wrapped in [ ]:
240	* - Parts are separated by "\|" and expanded individually.
241	* - Supported parts:
242	* - "x-y" → Integer range (inclusive, max 1000 values)
243	* - "_TABLE:..." → Lookup values from a TCA table
244	* - Otherwise → Literal value
245	*/
246	private function expandParameters(array $paramArray, int $pid): array
247	{
248	/** @var array<string, string\|int\|float\|null> $paramArray */
249	foreach ($paramArray as $parameter => $rawValue) {	15✔
250	$value = trim((string) $rawValue);	9✔
251
252	if ($this->isWrappedInSquareBrackets($value)) {	9✔
253	/** @var array<int, string\|int> $expanded */
254	$expanded = $this->expandBracketedValue($value, $pid, $parameter, $paramArray);	9✔
255	$paramArray[$parameter] = $expanded;	9✔
256	} else {
257	/** @var array<int, string> $paramArray */
258	$paramArray[$parameter] = [$value];	7✔
259	}
260	}
261
262	/** @var array<string, array<int, string\|int>> $paramArray */
263	return $paramArray;	15✔
264	}
265
266	/**
267	* Expands a value wrapped in [ ] into an array of individual values.
268	*
269	* @param array<string, mixed> $paramArray
270	* @return array<int, string\|int>
271	*/
272	private function expandBracketedValue(string $value, int $pid, string $parameter, array $paramArray): array
273	{
274	$expandedValues = [];	9✔
275	$innerValue = substr($value, 1, -1);	9✔
276	$parts = explode('\|', $innerValue);	9✔
277
278	foreach ($parts as $part) {	9✔
279	$part = trim($part);	9✔
280
281	if ($this->isIntegerRange($part)) {	9✔
282	$expandedValues = array_merge($expandedValues, $this->expandIntegerRange($part));	4✔
283	} elseif (str_starts_with($part, '_TABLE:')) {	5✔
284	$expandedValues = array_merge(	1✔
285	$expandedValues,	1✔
286	$this->expandTableLookup($part, $pid, $parameter, $paramArray)	1✔
287	);	1✔
288	} else {
289	$expandedValues[] = $part;	4✔
290	}
291
292	// Allow custom hooks to modify the expanded values
293	$paramArray = $this->runExpandParametersHook($paramArray, $parameter, $part, $pid);	9✔
294	}
295
296	return array_values(array_unique($expandedValues));	9✔
297	}
298
299	/**
300	* Checks whether a part is an integer range like "1-34" or "-40--30".
301	*/
302	private function isIntegerRange(string $part): bool
303	{
304	return (bool) preg_match('/^(-?\d+)\s-\s(-?\d+)$/', $part);	9✔
305	}
306
307	/**
308	* Expands an integer range string into an array of integers.
309	*
310	* @return array<int, int>
311	*/
312	private function expandIntegerRange(string $part): array
313	{
314	preg_match('/^(-?\d+)\s-\s(-?\d+)$/', $part, $matches);	4✔
315
316	[, $start, $end] = $matches;	4✔
317	$start = (int) $start;	4✔
318	$end = (int) $end;	4✔
319
320	if ($start > $end) {	4✔
321	[$start, $end] = [$end, $start];	1✔
322	}
323
324	$range = range($start, $end);	4✔
325
326	// Safety limit to prevent runaway ranges
327	return count($range) > 1000 ? array_slice($range, 0, 1000) : $range;	4✔
328	}
329
330	/**
331	* Expands a _TABLE lookup definition into values from the TCA table.
332	*
333	* @param array<string, mixed> $paramArray
334	* @return array<int, string\|int>
335	*/
336	private function expandTableLookup(string $definition, int $pid, string $parameter, array $paramArray): array
337	{
338	$subparts = GeneralUtility::trimExplode(';', $definition);	1✔
339	$options = [];	1✔
340
341	foreach ($subparts as $subpart) {	1✔
342	[$key, $value] = GeneralUtility::trimExplode(':', $subpart);	1✔
343	$options[$key] = $value;	1✔
344	}
345
346	if (!isset($GLOBALS['TCA'][$options['_TABLE']])) {	1✔
347	return []; // invalid table	1✔
348	}
349
NEW 350	return $this->extractParamsFromCustomTable($options, $pid, $paramArray, $parameter);	×
351	}
352
353	private function isWrappedInSquareBrackets(string $string): bool
354	{
355	return str_starts_with($string, '[') && str_ends_with($string, ']');	9✔
356	}
357
358	/**
359	* @return BackendUserAuthentication
360	*/
361	private function getBackendUser()
362	{
363	// Make sure the _cli_ user is loaded
364	Bootstrap::initializeBackendAuthentication();	1✔
365	if ($this->backendUser === null) {	1✔
366	$this->backendUser = $GLOBALS['BE_USER'];	1✔
367	}
368	return $this->backendUser;	1✔
369	}
370
371	private function getQueryBuilder(string $table): QueryBuilder
372	{
373	return GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);	×
374	}
375
376	/**
377	* @psalm-param array-key $parameter
378	*/
379	private function runExpandParametersHook(array $paramArray, int\|string $parameter, string $path, int $pid): array
380	{
381	if (is_array(	9✔
382	$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] ?? null	9✔
383	)) {	9✔
384	$_params = [	×
385	'pObj' => &$this,	×
386	'paramArray' => &$paramArray,	×
387	'currentKey' => $parameter,	×
388	'currentValue' => $path,	×
389	'pid' => $pid,	×
390	];	×
391	foreach ($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['crawler/class.tx_crawler_lib.php']['expandParameters'] as $_funcRef) {	×
392	GeneralUtility::callUserFunction($_funcRef, $_params, $this);	×
393	}
394	}
395	return $paramArray;	9✔
396	}
397
398	private function getPidArray(int $recursiveDepth, int $lookUpPid): array
399	{
400	if ($recursiveDepth > 0) {	×
401	$pageRepository = GeneralUtility::makeInstance(PageRepository::class);	×
402	$pidArray = $pageRepository->getPageIdsRecursive([$lookUpPid], $recursiveDepth);	×
403	} else {
404	$pidArray = [$lookUpPid];	×
405	}
406	return $pidArray;	×
407	}
408
409	private function expandPidList(array $treeCache, int $pid, int $depth, PageTreeView $tree, array $pidList): array
410	{
411	if (empty($treeCache[$pid][$depth])) {	×
412	$tree->reset();	×
413	$tree->getTree($pid, $depth);	×
414	$treeCache[$pid][$depth] = $tree->tree;	×
415	}
416
417	foreach ($treeCache[$pid][$depth] as $data) {	×
418	$pidList[] = (int) $data['row']['uid'];	×
419	}
420	return $pidList;	×
421	}
422
423	private function extractParamsFromCustomTable(
424	array $subpartParams,
425	int $pid,
426	array $paramArray,
427	int\|string $parameter
428	): array {
429	$lookUpPid = isset($subpartParams['_PID']) ? (int) $subpartParams['_PID'] : $pid;	×
430	$recursiveDepth = isset($subpartParams['_RECURSIVE']) ? (int) $subpartParams['_RECURSIVE'] : 0;	×
431	$pidField = isset($subpartParams['_PIDFIELD']) ? trim((string) $subpartParams['_PIDFIELD']) : 'pid';	×
432	$where = $subpartParams['_WHERE'] ?? '';	×
433	$addTable = $subpartParams['_ADDTABLE'] ?? '';	×
434
435	$fieldName = ($subpartParams['_FIELD'] ?? '') ?: 'uid';	×
436	if ($fieldName === 'uid' \|\| $GLOBALS['TCA'][$subpartParams['_TABLE']]['columns'][$fieldName]) {	×
437	$queryBuilder = $this->getQueryBuilder($subpartParams['_TABLE']);	×
438	$pidArray = $this->getPidArray($recursiveDepth, $lookUpPid);	×
439
440	$queryBuilder->getRestrictions()	×
441	->removeAll()	×
442	->add(GeneralUtility::makeInstance(DeletedRestriction::class));	×
443
444	$queryBuilder	×
445	->select($fieldName)	×
446	->from($subpartParams['_TABLE'])	×
447	->where(	×
448	$queryBuilder->expr()->in(	×
449	$pidField,	×
450	$queryBuilder->createNamedParameter($pidArray, ArrayParameterType::INTEGER)	×
451	),	×
452	$where	×
453	);	×
454
455	if (!empty($addTable)) {	×
456	// TODO: Check if this works as intended!
457	$addTables = GeneralUtility::trimExplode(',', $addTable, true);	×
458	foreach ($addTables as $table) {	×
459	$queryBuilder->from($table);	×
460	}
461	}
462	$transOrigPointerField = $GLOBALS['TCA'][$subpartParams['_TABLE']]['ctrl']['transOrigPointerField'] ?? false;	×
463
464	if (($subpartParams['_ENABLELANG'] ?? false) && $transOrigPointerField) {	×
465	$queryBuilder->andWhere($queryBuilder->expr()->lte($transOrigPointerField, 0));	×
466	}
467
468	$statement = $queryBuilder->executeQuery();	×
469
470	$rows = [];	×
471	while ($row = $statement->fetchAssociative()) {	×
472	$rows[$row[$fieldName]] = $row;	×
473	}
474
475	if (is_array($rows)) {	×
476	$paramArray[$parameter] = array_merge($paramArray[$parameter], array_keys($rows));	×
477	}
478	}
479	return $paramArray;	×
480	}
481	}

tomasnorre / crawler / 17555324436

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous