13166490870

Committed 05 Feb 2025 09:01PM UTC coverage: 67.753% (-1.4%) from 69.194%

Build # 13166490870

Build Type

push

github

Committed by

web-flow

Commit Message

[TASK] Refactor commands (#1127)

Run Details

42 of 75 new or added lines in 2 files covered. (56.0%)

19 existing lines in 2 files now uncovered.

1830 of 2701 relevant lines covered (67.75%)

3.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.26

/Classes/QueueExecutor.php

<?php

declare(strict_types=1);

namespace AOE\Crawler;

/*
 * (c) 2022 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
 *
 * This file is part of the TYPO3 Crawler Extension.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use AOE\Crawler\Controller\CrawlerController;
use AOE\Crawler\Converter\JsonCompatibilityConverter;
use AOE\Crawler\CrawlStrategy\CallbackExecutionStrategy;
use AOE\Crawler\CrawlStrategy\CrawlStrategyFactory;
use AOE\Crawler\CrawlStrategy\CrawlStrategyInterface;
use AOE\Crawler\Event\AfterUrlCrawledEvent;
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
use TYPO3\CMS\Core\Http\Uri;
use TYPO3\CMS\Core\SingletonInterface;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * Fetches a URL based on the selected strategy or via a callback.
 * @internal since v9.2.5
 */
class QueueExecutor implements SingletonInterface
{
    protected CrawlStrategyInterface $crawlStrategy;

    public function __construct(
        CrawlStrategyFactory $crawlStrategyFactory,
        private readonly EventDispatcher $eventDispatcher
    ) {
        $this->crawlStrategy = $crawlStrategyFactory->create();
    }

    /**
     * Takes a queue record and fetches the contents of the URL.
     * In the future, updating the queue item & additional signal/slot/events should also happen in here.
     *
     * @return array|bool|mixed|string
     */
    public function executeQueueItem(array $queueItem, CrawlerController $crawlerController)
    {
        $parameters = '';
        if (isset($queueItem['parameters'])) {
            // Decode parameters:
            /** @var JsonCompatibilityConverter $jsonCompatibleConverter */
            $jsonCompatibleConverter = GeneralUtility::makeInstance(JsonCompatibilityConverter::class);
            $parameters = $jsonCompatibleConverter->convert($queueItem['parameters']);
        }

        if (!is_array($parameters) || empty($parameters)) {
            return 'ERROR';
        }
        if (isset($parameters['_CALLBACKOBJ'])) {
            $className = $parameters['_CALLBACKOBJ'];
            unset($parameters['_CALLBACKOBJ']);
            $result = GeneralUtility::makeInstance(CallbackExecutionStrategy::class)
                ->fetchByCallback($className, $parameters, $crawlerController);
            $result = [
                'content' => json_encode($result),
            ];
        } else {
            // Regular FE request
            $crawlerId = $this->generateCrawlerIdFromQueueItem($queueItem);

            $url = new Uri($parameters['url']);
            $result = $this->crawlStrategy->fetchUrlContents($url, $crawlerId);
            if ($result !== false) {
                $result = [
                    'content' => json_encode($result),
                ];
                $this->eventDispatcher->dispatch(new AfterUrlCrawledEvent($parameters['url'], $result));
            }
        }
        return $result;
    }

    protected function generateCrawlerIdFromQueueItem(array $queueItem): string
    {
        return $queueItem['qid'] . ':' . md5(
            $queueItem['qid'] . '|' . $queueItem['set_id'] . '|' . $GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']
        );
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace AOE\Crawler;
6
7	/*
8	* (c) 2022 Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9	*
10	* This file is part of the TYPO3 Crawler Extension.
11	*
12	* It is free software; you can redistribute it and/or modify it under
13	* the terms of the GNU General Public License, either version 2
14	* of the License, or any later version.
15	*
16	* For the full copyright and license information, please read the
17	* LICENSE.txt file that was distributed with this source code.
18	*
19	* The TYPO3 project - inspiring people to share!
20	*/
21
22	use AOE\Crawler\Controller\CrawlerController;
23	use AOE\Crawler\Converter\JsonCompatibilityConverter;
24	use AOE\Crawler\CrawlStrategy\CallbackExecutionStrategy;
25	use AOE\Crawler\CrawlStrategy\CrawlStrategyFactory;
26	use AOE\Crawler\CrawlStrategy\CrawlStrategyInterface;
27	use AOE\Crawler\Event\AfterUrlCrawledEvent;
28	use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
29	use TYPO3\CMS\Core\Http\Uri;
30	use TYPO3\CMS\Core\SingletonInterface;
31	use TYPO3\CMS\Core\Utility\GeneralUtility;
32
33	/**
34	* Fetches a URL based on the selected strategy or via a callback.
35	* @internal since v9.2.5
36	*/
37	class QueueExecutor implements SingletonInterface
38	{
39	protected CrawlStrategyInterface $crawlStrategy;
40
41	public function __construct(
42	CrawlStrategyFactory $crawlStrategyFactory,
43	private readonly EventDispatcher $eventDispatcher
44	) {
45	$this->crawlStrategy = $crawlStrategyFactory->create();	21✔
46	}
47
48	/**
49	* Takes a queue record and fetches the contents of the URL.
50	* In the future, updating the queue item & additional signal/slot/events should also happen in here.
51	*
52	* @return array\|bool\|mixed\|string
53	*/
54	public function executeQueueItem(array $queueItem, CrawlerController $crawlerController)
55	{
56	$parameters = '';	7✔
57	if (isset($queueItem['parameters'])) {	7✔
58	// Decode parameters:
59	/** @var JsonCompatibilityConverter $jsonCompatibleConverter */
60	$jsonCompatibleConverter = GeneralUtility::makeInstance(JsonCompatibilityConverter::class);	6✔
61	$parameters = $jsonCompatibleConverter->convert($queueItem['parameters']);	6✔
62	}
63
64	if (!is_array($parameters) \|\| empty($parameters)) {	7✔
65	return 'ERROR';	6✔
66	}
67	if (isset($parameters['_CALLBACKOBJ'])) {	1✔
68	$className = $parameters['_CALLBACKOBJ'];	1✔
69	unset($parameters['_CALLBACKOBJ']);	1✔
70	$result = GeneralUtility::makeInstance(CallbackExecutionStrategy::class)	1✔
71	->fetchByCallback($className, $parameters, $crawlerController);	1✔
72	$result = [	1✔
73	'content' => json_encode($result),	1✔
74	];	1✔
75	} else {
76	// Regular FE request
UNCOV 77	$crawlerId = $this->generateCrawlerIdFromQueueItem($queueItem);	×
78
UNCOV 79	$url = new Uri($parameters['url']);	×
UNCOV 80	$result = $this->crawlStrategy->fetchUrlContents($url, $crawlerId);	×
UNCOV 81	if ($result !== false) {	×
82	$result = [	×
83	'content' => json_encode($result),	×
84	];	×
85	$this->eventDispatcher->dispatch(new AfterUrlCrawledEvent($parameters['url'], $result));	×
86	}
87	}
88	return $result;	1✔
89	}
90
91	protected function generateCrawlerIdFromQueueItem(array $queueItem): string
92	{
UNCOV 93	return $queueItem['qid'] . ':' . md5(	×
UNCOV 94	$queueItem['qid'] . '\|' . $queueItem['set_id'] . '\|' . $GLOBALS['TYPO3_CONF_VARS']['SYS']['encryptionKey']	×
UNCOV 95	);	×
96	}
97	}

tomasnorre / crawler / 13166490870

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous