39921988843

Committed 23 Oct 2026 08:03AM UTC coverage: 69.663% (-0.2%) from 69.891%

Build # 39921988843

Build Type

Pull #1314

github

Committed by

web-flow

Commit Message

Merge 528277a59 into 23a13b0d1

Pull Request Pull Request #1314: [TASK] Pin dev-dependencies

Coverage Stats

2 of 2 new or added lines in 1 file covered. (100.0%)

37 existing lines in 3 files now uncovered.

1922 of 2759 relevant lines covered (69.66%)

3.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

84.76

/Classes/Command/ProcessQueueCommand.php

<?php

declare(strict_types=1);

namespace AOE\Crawler\Command;

/*
 * (c) 2021 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
 *
 * This file is part of the TYPO3 Crawler Extension.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
use AOE\Crawler\Controller\CrawlerController;
use AOE\Crawler\Crawler;
use AOE\Crawler\Domain\Model\Process;
use AOE\Crawler\Domain\Repository\ProcessRepository;
use AOE\Crawler\Domain\Repository\QueueRepository;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * @internal since v12.0.0
 */
#[AsCommand(
    name: 'crawler:processQueue',
    description: 'Process queue entries',
)]
class ProcessQueueCommand extends Command
{
    private const CLI_STATUS_NOTHING_PROCCESSED = 0;
    private const CLI_STATUS_REMAIN = 1;
    private const CLI_STATUS_PROCESSED = 2;
    private const CLI_STATUS_ABORTED = 4;
    private readonly string $processId;
    private array $extensionSettings;

    public function __construct(
        private readonly Crawler $crawler,
        private readonly CrawlerController $crawlerController,
        private readonly ProcessRepository $processRepository,
        private readonly QueueRepository $queueRepository,
    ) {
        $this->processId = md5(microtime() . random_bytes(12));
        parent::__construct();
    }

    /**
     * Crawler Command - Crawling the URLs from the queue
     *
     * Examples:
     *
     * --- Will trigger the crawler which starts to process the queue entries
     * $ typo3 crawler:crawlQueue
     */
    #[\Override]
    public function execute(InputInterface $input, OutputInterface $output): int
    {
        $amount = $input->getOption('amount');
        $sleeptime = $input->getOption('sleeptime');
        $sleepafter = $input->getOption('sleepafter');

        $this->extensionSettings = $this->getExtensionSettings();

        $result = self::CLI_STATUS_NOTHING_PROCCESSED;

        if (!$this->crawler->isDisabled() && $this->checkAndAcquireNewProcess($this->processId)) {
            $countInARun = $amount ? (int) $amount : (int) $this->extensionSettings['countInARun'];
            $sleepAfterFinish = $sleepafter ? (int) $sleepafter : (int) $this->extensionSettings['sleepAfterFinish'];
            $sleepTime = $sleeptime ? (int) $sleeptime : (int) $this->extensionSettings['sleepTime'];

            try {
                // Run process:
                $result = $this->runProcess($countInARun, $sleepTime, $sleepAfterFinish);
            } catch (\Throwable $e) {
                $output->writeln('<warning>' . $e::class . ': ' . $e->getMessage() . '</warning>');
                $result = self::CLI_STATUS_ABORTED;
            }

            // Cleanup
            $this->processRepository->deleteProcessesWithoutItemsAssigned();
            $this->processRepository->markRequestedProcessesAsNotActive([$this->processId]);
            $this->queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries([$this->processId]);

            $output->writeln(
                '<info>Unprocessed Items remaining:' . count(
                    $this->queueRepository->getUnprocessedItems()
                ) . ' (' . $this->processId . ')</info>'
            );
            $result |= (count(
                $this->queueRepository->getUnprocessedItems()
            ) > 0 ? self::CLI_STATUS_REMAIN : self::CLI_STATUS_NOTHING_PROCCESSED);
        } else {
            $result |= self::CLI_STATUS_ABORTED;
        }

        $output->writeln((string) $result);
        return $result & self::CLI_STATUS_ABORTED;
    }

    #[\Override]
    protected function configure(): void
    {
        $this->setDescription('Trigger the crawler to process the queue entries');

        $this->setHelp(
            'Crawler Command - Crawling the URLs from the queue' . chr(10) . chr(10) .
            '
            Examples:
              --- Will trigger the crawler which starts to process the queue entries
              $ typo3 crawler:processqueue --amount 15 --sleepafter 5 --sleeptime 2
            '
        );
        $this->addOption(
            'amount',
            '',
            InputOption::VALUE_OPTIONAL,
            'How many pages should be crawled during that run',
            '0'
        );

        $this->addOption(
            'sleepafter',
            '',
            InputOption::VALUE_OPTIONAL,
            'Amount of seconds which the system should use to relax after all crawls are done',
            '0'
        );

        $this->addOption(
            'sleeptime',
            '',
            InputOption::VALUE_OPTIONAL,
            'Amount of microseconds which the system should use to relax between crawls'
        );
    }

    /**
     * Running the functionality of the CLI (crawling URLs from queue)
     */
    private function runProcess(int $countInARun, int $sleepTime, int $sleepAfterFinish): int
    {
        $result = 0;
        $counter = 0;

        // Clean up the queue
        $this->queueRepository->cleanupQueue();

        // Select entries:
        $records = $this->queueRepository->fetchRecordsToBeCrawled($countInARun);

        if (!empty($records)) {
            $quidList = [];

            foreach ($records as $record) {
                $quidList[] = $record['qid'];
            }

            //save the number of assigned queue entries to determine how many have been processed later
            $numberOfAffectedRows = $this->queueRepository->updateProcessIdAndSchedulerForQueueIds(
                $quidList,
                $this->processId
            );
            $this->processRepository->updateProcessAssignItemsCount($numberOfAffectedRows, $this->processId);

            if ($numberOfAffectedRows !== count($quidList)) {
                return $result | self::CLI_STATUS_ABORTED;
            }

            foreach ($records as $record) {
                $result |= $this->crawlerController->readUrl($record['qid'], false, $this->processId);

                $counter++;
                // Just to relax the system
                usleep($sleepTime);

                // if during the start and the current read url the cli has been disable we need to return from the function
                // mark the process NOT as ended.
                if ($this->crawler->isDisabled()) {
                    return $result | self::CLI_STATUS_ABORTED;
                }

                if (!$this->processRepository->isProcessActive($this->processId)) {
                    $result |= self::CLI_STATUS_ABORTED;
                    //possible timeout
                    break;
                }
            }

            sleep($sleepAfterFinish);
        }

        if ($counter > 0) {
            $result |= self::CLI_STATUS_PROCESSED;
        }

        return $result;
    }

    /**
     * Try to acquire a new process with the given id
     * also performs some auto-cleanup for orphan processes
     */
    private function checkAndAcquireNewProcess(string $id): bool
    {
        $returnValue = true;

        $systemProcessId = getmypid();
        if (!$systemProcessId) {
            return false;
        }

        $processCount = 0;
        $orphanProcesses = [];

        $activeProcesses = $this->processRepository->findAllActive();

        /** @var Process $process */
        foreach ($activeProcesses as $process) {
            if ($process->getTtl() < time()) {
                $orphanProcesses[] = $process->getProcessId();
            } else {
                $processCount++;
            }
        }

        // if there are less than allowed active processes then add a new one
        if ($processCount < (int) $this->extensionSettings['processLimit']) {
            $this->processRepository->addProcess($id, $systemProcessId);
        } else {
            $returnValue = false;
        }

        $this->processRepository->deleteProcessesMarkedAsDeleted();
        $this->processRepository->markRequestedProcessesAsNotActive($orphanProcesses);
        $this->queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries($orphanProcesses);

        return $returnValue;
    }

    private function getExtensionSettings(): array
    {
        return GeneralUtility::makeInstance(ExtensionConfigurationProvider::class)->getExtensionConfiguration();
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace AOE\Crawler\Command;
6
7	/*
8	* (c) 2021 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9	*
10	* This file is part of the TYPO3 Crawler Extension.
11	*
12	* It is free software; you can redistribute it and/or modify it under
13	* the terms of the GNU General Public License, either version 2
14	* of the License, or any later version.
15	*
16	* For the full copyright and license information, please read the
17	* LICENSE.txt file that was distributed with this source code.
18	*
19	* The TYPO3 project - inspiring people to share!
20	*/
21
22	use AOE\Crawler\Configuration\ExtensionConfigurationProvider;
23	use AOE\Crawler\Controller\CrawlerController;
24	use AOE\Crawler\Crawler;
25	use AOE\Crawler\Domain\Model\Process;
26	use AOE\Crawler\Domain\Repository\ProcessRepository;
27	use AOE\Crawler\Domain\Repository\QueueRepository;
28	use Symfony\Component\Console\Attribute\AsCommand;
29	use Symfony\Component\Console\Command\Command;
30	use Symfony\Component\Console\Input\InputInterface;
31	use Symfony\Component\Console\Input\InputOption;
32	use Symfony\Component\Console\Output\OutputInterface;
33	use TYPO3\CMS\Core\Utility\GeneralUtility;
34
35	/**
36	* @internal since v12.0.0
37	*/
38	#[AsCommand(
UNCOV 39	name: 'crawler:processQueue',	×
UNCOV 40	description: 'Process queue entries',	×
UNCOV 41	)]	×
42	class ProcessQueueCommand extends Command
43	{
44	private const CLI_STATUS_NOTHING_PROCCESSED = 0;
45	private const CLI_STATUS_REMAIN = 1;
46	private const CLI_STATUS_PROCESSED = 2;
47	private const CLI_STATUS_ABORTED = 4;
48	private readonly string $processId;
49	private array $extensionSettings;
50
51	public function __construct(
52	private readonly Crawler $crawler,
53	private readonly CrawlerController $crawlerController,
54	private readonly ProcessRepository $processRepository,
55	private readonly QueueRepository $queueRepository,
56	) {
57	$this->processId = md5(microtime() . random_bytes(12));	2 ✔
58	parent::__construct();	2 ✔
59	}
60
61	/**
62	* Crawler Command - Crawling the URLs from the queue
63	*
64	* Examples:
65	*
66	* --- Will trigger the crawler which starts to process the queue entries
67	* $ typo3 crawler:crawlQueue
68	*/
69	#[\Override]
70	public function execute(InputInterface $input, OutputInterface $output): int
71	{
72	$amount = $input->getOption('amount');	2 ✔
73	$sleeptime = $input->getOption('sleeptime');	2 ✔
74	$sleepafter = $input->getOption('sleepafter');	2 ✔
75
76	$this->extensionSettings = $this->getExtensionSettings();	2 ✔
77
78	$result = self::CLI_STATUS_NOTHING_PROCCESSED;	2 ✔
79
80	if (!$this->crawler->isDisabled() && $this->checkAndAcquireNewProcess($this->processId)) {	2 ✔
81	$countInARun = $amount ? (int) $amount : (int) $this->extensionSettings['countInARun'];	2 ✔
82	$sleepAfterFinish = $sleepafter ? (int) $sleepafter : (int) $this->extensionSettings['sleepAfterFinish'];	2 ✔
83	$sleepTime = $sleeptime ? (int) $sleeptime : (int) $this->extensionSettings['sleepTime'];	2 ✔
84
85	try {
86	// Run process:
87	$result = $this->runProcess($countInARun, $sleepTime, $sleepAfterFinish);	2 ✔
UNCOV 88	} catch (\Throwable $e) {	×
UNCOV 89	$output->writeln('<warning>' . $e::class . ': ' . $e->getMessage() . '</warning>');	×
UNCOV 90	$result = self::CLI_STATUS_ABORTED;	×
91	}
92
93	// Cleanup
94	$this->processRepository->deleteProcessesWithoutItemsAssigned();	2 ✔
95	$this->processRepository->markRequestedProcessesAsNotActive([$this->processId]);	2 ✔
96	$this->queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries([$this->processId]);	2 ✔
97
98	$output->writeln(	2 ✔
99	'<info>Unprocessed Items remaining:' . count(	2 ✔
100	$this->queueRepository->getUnprocessedItems()	2 ✔
101	) . ' (' . $this->processId . ')</info>'	2 ✔
102	);	2 ✔
103	$result \|= (count(	2 ✔
104	$this->queueRepository->getUnprocessedItems()	2 ✔
105	) > 0 ? self::CLI_STATUS_REMAIN : self::CLI_STATUS_NOTHING_PROCCESSED);	2 ✔
106	} else {
UNCOV 107	$result \|= self::CLI_STATUS_ABORTED;	×
108	}
109
110	$output->writeln((string) $result);	2 ✔
111	return $result & self::CLI_STATUS_ABORTED;	2 ✔
112	}
113
114	#[\Override]
115	protected function configure(): void
116	{
117	$this->setDescription('Trigger the crawler to process the queue entries');	2 ✔
118
119	$this->setHelp(	2 ✔
120	'Crawler Command - Crawling the URLs from the queue' . chr(10) . chr(10) .	2 ✔
121	'	2 ✔
122	Examples:
123	--- Will trigger the crawler which starts to process the queue entries
124	$ typo3 crawler:processqueue --amount 15 --sleepafter 5 --sleeptime 2
125	'	2 ✔
126	);	2 ✔
127	$this->addOption(	2 ✔
128	'amount',	2 ✔
129	'',	2 ✔
130	InputOption::VALUE_OPTIONAL,	2 ✔
131	'How many pages should be crawled during that run',	2 ✔
132	'0'	2 ✔
133	);	2 ✔
134
135	$this->addOption(	2 ✔
136	'sleepafter',	2 ✔
137	'',	2 ✔
138	InputOption::VALUE_OPTIONAL,	2 ✔
139	'Amount of seconds which the system should use to relax after all crawls are done',	2 ✔
140	'0'	2 ✔
141	);	2 ✔
142
143	$this->addOption(	2 ✔
144	'sleeptime',	2 ✔
145	'',	2 ✔
146	InputOption::VALUE_OPTIONAL,	2 ✔
147	'Amount of microseconds which the system should use to relax between crawls'	2 ✔
148	);	2 ✔
149	}
150
151	/**
152	* Running the functionality of the CLI (crawling URLs from queue)
153	*/
154	private function runProcess(int $countInARun, int $sleepTime, int $sleepAfterFinish): int
155	{
156	$result = 0;	2 ✔
157	$counter = 0;	2 ✔
158
159	// Clean up the queue
160	$this->queueRepository->cleanupQueue();	2 ✔
161
162	// Select entries:
163	$records = $this->queueRepository->fetchRecordsToBeCrawled($countInARun);	2 ✔
164
165	if (!empty($records)) {	2 ✔
166	$quidList = [];	2 ✔
167
168	foreach ($records as $record) {	2 ✔
169	$quidList[] = $record['qid'];	2 ✔
170	}
171
172	//save the number of assigned queue entries to determine how many have been processed later
173	$numberOfAffectedRows = $this->queueRepository->updateProcessIdAndSchedulerForQueueIds(	2 ✔
174	$quidList,	2 ✔
175	$this->processId	2 ✔
176	);	2 ✔
177	$this->processRepository->updateProcessAssignItemsCount($numberOfAffectedRows, $this->processId);	2 ✔
178
179	if ($numberOfAffectedRows !== count($quidList)) {	2 ✔
UNCOV 180	return $result \| self::CLI_STATUS_ABORTED;	×
181	}
182
183	foreach ($records as $record) {	2 ✔
184	$result \|= $this->crawlerController->readUrl($record['qid'], false, $this->processId);	2 ✔
185
186	$counter++;	2 ✔
187	// Just to relax the system
188	usleep($sleepTime);	2 ✔
189
190	// if during the start and the current read url the cli has been disable we need to return from the function
191	// mark the process NOT as ended.
192	if ($this->crawler->isDisabled()) {	2 ✔
UNCOV 193	return $result \| self::CLI_STATUS_ABORTED;	×
194	}
195
196	if (!$this->processRepository->isProcessActive($this->processId)) {	2 ✔
UNCOV 197	$result \|= self::CLI_STATUS_ABORTED;	×
198	//possible timeout
UNCOV 199	break;	×
200	}
201	}
202
203	sleep($sleepAfterFinish);	2 ✔
204	}
205
206	if ($counter > 0) {	2 ✔
207	$result \|= self::CLI_STATUS_PROCESSED;	2 ✔
208	}
209
210	return $result;	2 ✔
211	}
212
213	/**
214	* Try to acquire a new process with the given id
215	* also performs some auto-cleanup for orphan processes
216	*/
217	private function checkAndAcquireNewProcess(string $id): bool
218	{
219	$returnValue = true;	2 ✔
220
221	$systemProcessId = getmypid();	2 ✔
222	if (!$systemProcessId) {	2 ✔
UNCOV 223	return false;	×
224	}
225
226	$processCount = 0;	2 ✔
227	$orphanProcesses = [];	2 ✔
228
229	$activeProcesses = $this->processRepository->findAllActive();	2 ✔
230
231	/** @var Process $process */
232	foreach ($activeProcesses as $process) {	2 ✔
233	if ($process->getTtl() < time()) {	×
UNCOV 234	$orphanProcesses[] = $process->getProcessId();	×
235	} else {
UNCOV 236	$processCount++;	×
237	}
238	}
239
240	// if there are less than allowed active processes then add a new one
241	if ($processCount < (int) $this->extensionSettings['processLimit']) {	2 ✔
242	$this->processRepository->addProcess($id, $systemProcessId);	2 ✔
243	} else {
UNCOV 244	$returnValue = false;	×
245	}
246
247	$this->processRepository->deleteProcessesMarkedAsDeleted();	2 ✔
248	$this->processRepository->markRequestedProcessesAsNotActive($orphanProcesses);	2 ✔
249	$this->queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries($orphanProcesses);	2 ✔
250
251	return $returnValue;	2 ✔
252	}
253
254	private function getExtensionSettings(): array
255	{
256	return GeneralUtility::makeInstance(ExtensionConfigurationProvider::class)->getExtensionConfiguration();	2 ✔
257	}
258	}

tomasnorre / crawler / 39921988843

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous