16823073567

Committed 08 Aug 2025 05:55AM UTC coverage: 69.106% (+0.03%) from 69.08%

Build # 16823073567

Build Type

Pull #1150

github

Committed by

web-flow

Commit Message

Merge 54eeb2815 into 7c1763271

Pull Request Pull Request #1150: fix: ensure that pages are added to queue when page cache is cleared

Run Details

3 of 3 new or added lines in 1 file covered. (100.0%)

2 existing lines in 1 file now uncovered.

1870 of 2706 relevant lines covered (69.11%)

3.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.1

/Classes/Service/QueueService.php

<?php

declare(strict_types=1);

namespace AOE\Crawler\Service;

/*
 * (c) 2021 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
 *
 * This file is part of the TYPO3 Crawler Extension.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use AOE\Crawler\Controller\CrawlerController;
use TYPO3\CMS\Core\Domain\Repository\PageRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * @internal since v9.2.5
 */
class QueueService
{
    public function __construct(
        private CrawlerController $crawlerController
    )
    {
        if ($this->crawlerController->setID <= 0) {
            $this->crawlerController->setID = GeneralUtility::md5int(microtime());
        }
    }

    public function addPageToQueue(int $pageUid, int $time = 0): void
    {
        error_log('Test', 3, '/tmp/tomasnorre.log');
        $pageData = GeneralUtility::makeInstance(PageRepository::class)->getPage($pageUid, true);
        $configurations = $this->crawlerController->getUrlsForPageRow($pageData);
        // Currently this is only used from the DataHandlerHook, and we don't know of any allowed/disallowed configurations,
        // when clearing the cache, therefore we allow all configurations in this case.
        // This next lines could be skipped as it will return the incoming configurations, but for visibility and
        // later implementation it's kept as it do no harm.
        $allowedConfigurations = [];
        $configurations = ConfigurationService::removeDisallowedConfigurations($allowedConfigurations, $configurations);
        $downloadUrls = [];
        $duplicateTrack = [];

        if (is_array($configurations)) {
            foreach ($configurations as $configuration) {
                //enable inserting of entries
                $this->crawlerController->registerQueueEntriesInternallyOnly = false;
                $this->crawlerController->urlListFromUrlArray(
                    $configuration,
                    $pageData,
                    $time,
                    300,
                    true,
                    false,
                    $duplicateTrack,
                    $downloadUrls,
                    array_keys($this->getCrawlerProcInstructions())
                );

                //reset the queue because the entries have been written to the db
                unset($this->crawlerController->queueEntries);
            }
        }
    }

    /**
     * Reads the registered processingInstructions of the crawler
     */
    private function getCrawlerProcInstructions(): array
    {
        $crawlerProcInstructions = [];
        if (!empty($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['procInstructions'])) {
            foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['procInstructions'] as $configuration) {
                $crawlerProcInstructions[$configuration['key']] = $configuration['value'];
            }
        }

        return $crawlerProcInstructions;
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace AOE\Crawler\Service;
6
7	/*
8	* (c) 2021 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9	*
10	* This file is part of the TYPO3 Crawler Extension.
11	*
12	* It is free software; you can redistribute it and/or modify it under
13	* the terms of the GNU General Public License, either version 2
14	* of the License, or any later version.
15	*
16	* For the full copyright and license information, please read the
17	* LICENSE.txt file that was distributed with this source code.
18	*
19	* The TYPO3 project - inspiring people to share!
20	*/
21
22	use AOE\Crawler\Controller\CrawlerController;
23	use TYPO3\CMS\Core\Domain\Repository\PageRepository;
24	use TYPO3\CMS\Core\Utility\GeneralUtility;
25
26	/**
27	* @internal since v9.2.5
28	*/
29	class QueueService
30	{
31	public function __construct(
32	private CrawlerController $crawlerController
33	)
34	{
35	if ($this->crawlerController->setID <= 0) {	3✔
36	$this->crawlerController->setID = GeneralUtility::md5int(microtime());	3✔
37	}
38	}
39
40	public function addPageToQueue(int $pageUid, int $time = 0): void
41	{
42	error_log('Test', 3, '/tmp/tomasnorre.log');	3✔
43	$pageData = GeneralUtility::makeInstance(PageRepository::class)->getPage($pageUid, true);	3✔
44	$configurations = $this->crawlerController->getUrlsForPageRow($pageData);	3✔
45	// Currently this is only used from the DataHandlerHook, and we don't know of any allowed/disallowed configurations,
46	// when clearing the cache, therefore we allow all configurations in this case.
47	// This next lines could be skipped as it will return the incoming configurations, but for visibility and
48	// later implementation it's kept as it do no harm.
49	$allowedConfigurations = [];	3✔
50	$configurations = ConfigurationService::removeDisallowedConfigurations($allowedConfigurations, $configurations);	3✔
51	$downloadUrls = [];	3✔
52	$duplicateTrack = [];	3✔
53
54	if (is_array($configurations)) {	3✔
55	foreach ($configurations as $configuration) {	3✔
56	//enable inserting of entries
57	$this->crawlerController->registerQueueEntriesInternallyOnly = false;	3✔
58	$this->crawlerController->urlListFromUrlArray(	3✔
59	$configuration,	3✔
60	$pageData,	3✔
61	$time,	3✔
62	300,	3✔
63	true,	3✔
64	false,	3✔
65	$duplicateTrack,	3✔
66	$downloadUrls,	3✔
67	array_keys($this->getCrawlerProcInstructions())	3✔
68	);	3✔
69
70	//reset the queue because the entries have been written to the db
71	unset($this->crawlerController->queueEntries);	3✔
72	}
73	}
74	}
75
76	/**
77	* Reads the registered processingInstructions of the crawler
78	*/
79	private function getCrawlerProcInstructions(): array
80	{
81	$crawlerProcInstructions = [];	3✔
82	if (!empty($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['procInstructions'])) {	3✔
UNCOV 83	foreach ($GLOBALS['TYPO3_CONF_VARS']['EXTCONF']['crawler']['procInstructions'] as $configuration) {	×
UNCOV 84	$crawlerProcInstructions[$configuration['key']] = $configuration['value'];	×
85	}
86	}
87
88	return $crawlerProcInstructions;	3✔
89	}
90	}

tomasnorre / crawler / 16823073567

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous