11237471329

Committed 08 Oct 2024 02:20PM UTC coverage: 68.586% (-1.3%) from 69.862%

Build # 11237471329

Build Type

push

github

Committed by

web-flow

Commit Message

ci: Update coveralls workflow (#1109)

Run Details

1834 of 2674 relevant lines covered (68.59%)

3.37 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/Classes/Controller/Backend/BackendModuleStartCrawlingController.php

<?php

declare(strict_types=1);

namespace AOE\Crawler\Controller\Backend;

/*
 * (c) 2022-     Tomas Norre Mikkelsen <tomasnorre@gmail.com>
 *
 * This file is part of the TYPO3 Crawler Extension.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
use AOE\Crawler\Controller\CrawlerController;
use AOE\Crawler\Domain\Model\Reason;
use AOE\Crawler\Event\InvokeQueueChangeEvent;
use AOE\Crawler\Utility\MessageUtility;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use Symfony\Contracts\Service\Attribute\Required;
use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
use TYPO3\CMS\Backend\Routing\UriBuilder;
use TYPO3\CMS\Backend\Template\ModuleTemplate;
use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
use TYPO3\CMS\Core\Http\Uri;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * @internal since v12.0.0
 */
class BackendModuleStartCrawlingController extends AbstractBackendModuleController implements BackendModuleControllerInterface
{
    private const BACKEND_MODULE = 'web_site_crawler_start';
    private const LINE_FEED = 10;
    private const CARRIAGE_RETURN = 13;
    private int $reqMinute = 1000;
    private EventDispatcher $eventDispatcher;

    /**
     * @var array holds the selection of configuration from the configuration selector box
     */
    private $incomingConfigurationSelection = [];

    public function __construct(
        private readonly CrawlerController $crawlerController,
        private readonly UriBuilder $backendUriBuilder,
    ) {
    }

    #[Required]
    public function setEventDispatcher(): void
    {
        $this->eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);
    }

    public function handleRequest(ServerRequestInterface $request): ResponseInterface
    {
        $this->makeCrawlerProcessableChecks($this->extensionSettings);
        $this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);
        $this->moduleTemplate = $this->setupView($request, $this->pageUid);
        $this->moduleTemplate->makeDocHeaderModuleMenu([
            'id' => $request->getQueryParams()['id'] ?? -1,
        ]);

        $this->assignValues($request);
        return $this->moduleTemplate->renderResponse('Backend/ShowCrawlerInformation');
    }

    private function assignValues(ServerRequestInterface $request): ModuleTemplate
    {
        $logUrl = $this->backendUriBuilder->buildUriFromRoute('web_site_crawler_log', [
            'id' => $this->pageUid,
        ]);

        $crawlingDepth = $request->getParsedBody()['crawlingDepth'] ?? $request->getQueryParams()['crawlingDepth'] ?? '0';
        $crawlParameter = $request->getParsedBody()['_crawl'] ?? $request->getQueryParams()['_crawl'] ?? null;
        $downloadParameter = $request->getParsedBody()['_download'] ?? $request->getQueryParams()['_download'] ?? null;
        $scheduledTime = $this->getScheduledTime(
            (string) ($request->getParsedBody()['tstamp'] ?? $request->getQueryParams()['tstamp'] ?? null)
        );
        $submitCrawlUrls = isset($crawlParameter);
        $downloadCrawlUrls = isset($downloadParameter);

        $this->incomingConfigurationSelection = $request->getParsedBody()['configurationSelection'] ?? $request->getQueryParams()['configurationSelection'] ?? null;
        $this->incomingConfigurationSelection = is_array(
            $this->incomingConfigurationSelection
        ) ? $this->incomingConfigurationSelection : [];

        //$this->crawlerController = $this->getCrawlerController();
        $this->crawlerController->setID = GeneralUtility::md5int(microtime());

        $queueRows = '';
        $noConfigurationSelected = empty($this->incomingConfigurationSelection)
            || (count($this->incomingConfigurationSelection) === 1 && empty($this->incomingConfigurationSelection[0]));
        if ($noConfigurationSelected) {
            MessageUtility::addWarningMessage(
                $this->getLanguageService()->sL(
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.noConfigSelected'
                )
            );
        } else {
            if ($submitCrawlUrls) {
                $reason = new Reason();
                $reason->setReason(Reason::REASON_GUI_SUBMIT);
                $reason->setDetailText(
                    'The user ' . $GLOBALS['BE_USER']->user['username'] . ' added pages to the crawler queue manually'
                );
                $this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
            }

            $queueRows = $this->crawlerController->getPageTreeAndUrls(
                $this->pageUid,
                $crawlingDepth,
                $scheduledTime,
                $this->reqMinute,
                $submitCrawlUrls,
                $downloadCrawlUrls,
                // Do not filter any processing instructions
                [],
                $this->incomingConfigurationSelection
            );
        }

        // Download Urls to crawl:
        $downloadUrls = $this->crawlerController->downloadUrls;
        if ($downloadCrawlUrls) {
            // Creating output header:
            header('Content-Type: application/octet-stream');
            header('Content-Disposition: attachment; filename=CrawlerUrls.txt');

            // Printing the content of the CSV lines:
            echo implode(chr(self::CARRIAGE_RETURN) . chr(self::LINE_FEED), $downloadUrls);
            exit;
        }

        return $this->moduleTemplate->assignMultiple([
            'currentPageId' => $this->pageUid,
            'noConfigurationSelected' => $noConfigurationSelected,
            'submitCrawlUrls' => $submitCrawlUrls,
            'amountOfUrls' => count($this->crawlerController->duplicateTrack ?? []),
            'selectors' => $this->generateConfigurationSelectors($this->pageUid, $crawlingDepth, $request),
            'queueRows' => $queueRows,
            'displayActions' => 0,
            'actionUrl' => $this->getActionUrl(),
            'logUrl' => $logUrl,
        ]);
    }

    /**
     * Generates the configuration selectors for compiling URLs:
     */
    private function generateConfigurationSelectors(
        int $pageId,
        string $crawlingDepth,
        ServerRequestInterface $request
    ): array {
        $selectors = [];
        $selectors['depth'] = $this->selectorBox(
            [
                0 => $this->getLanguageService()->sL(
                    'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_0'
                ),
                1 => $this->getLanguageService()->sL(
                    'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_1'
                ),
                2 => $this->getLanguageService()->sL(
                    'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_2'
                ),
                3 => $this->getLanguageService()->sL(
                    'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_3'
                ),
                4 => $this->getLanguageService()->sL(
                    'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_4'
                ),
                99 => $this->getLanguageService()->sL(
                    'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_infi'
                ),
            ],
            'crawlingDepth',
            $crawlingDepth,
            false
        );

        // Configurations
        $availableConfigurations = $this->crawlerController->getConfigurationsForBranch(
            $pageId,
            (int) $crawlingDepth,
        );
        $selectors['configurations'] = $this->selectorBox(
            empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations),
            'configurationSelection',
            $this->incomingConfigurationSelection,
            true
        );

        // Scheduled time:
        $selectors['scheduled'] = $this->selectorBox(
            [
                'now' => $this->getLanguageService()->sL(
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.time.now'
                ),
                'midnight' => $this->getLanguageService()->sL(
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.time.midnight'
                ),
                '04:00' => $this->getLanguageService()->sL(
                    'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.time.4am'
                ),
            ],
            'tstamp',
            $request->getParsedBody()['tstamp'] ?? null,
            false
        );

        return $selectors;
    }

    /**
     * Create selector box
     *
     * @param array $optArray Options key(value) => label pairs
     * @param string $name Selector box name
     * @param string|array|null $value Selector box value (array for multiple...)
     * @param boolean $multiple If set, will draw multiple box.
     *
     * @return string HTML select element
     */
    private function selectorBox($optArray, $name, string|array|null $value, bool $multiple): string
    {
        if (!is_string($value) || !is_array($value)) {
            $value = '';
        }

        $options = [];
        foreach ($optArray as $key => $val) {
            $selected = (!$multiple && !strcmp($value, (string) $key)) || ($multiple && in_array(
                $key,
                (array) $value,
                true
            ));
            $options[] = '
                <option value="' . $key . '" ' . ($selected ? ' selected="selected"' : '') . '>' . htmlspecialchars(
                (string) $val,
                ENT_QUOTES | ENT_HTML5
            ) . '</option>';
        }

        return '<select class="form-select" name="' . htmlspecialchars(
            $name . ($multiple ? '[]' : ''),
            ENT_QUOTES | ENT_HTML5
        ) . '"' . ($multiple ? ' multiple' : '') . '>' . implode('', $options) . '</select>';
    }

    private function getScheduledTime(string $time): int
    {
        $scheduledTime = match ($time) {
            'midnight' => mktime(0, 0, 0),
            '04:00' => mktime(0, 0, 0) + 4 * 3600,
            default => time(),
        };

        if (!$scheduledTime) {
            return time();
        }

        return $scheduledTime;
    }

    /**
     * @throws RouteNotFoundException
     */
    private function getActionUrl(): Uri
    {
        return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace AOE\Crawler\Controller\Backend;
6
7	/*
8	* (c) 2022- Tomas Norre Mikkelsen <tomasnorre@gmail.com>
9	*
10	* This file is part of the TYPO3 Crawler Extension.
11	*
12	* It is free software; you can redistribute it and/or modify it under
13	* the terms of the GNU General Public License, either version 2
14	* of the License, or any later version.
15	*
16	* For the full copyright and license information, please read the
17	* LICENSE.txt file that was distributed with this source code.
18	*
19	* The TYPO3 project - inspiring people to share!
20	*/
21
22	use AOE\Crawler\Controller\Backend\Helper\UrlBuilder;
23	use AOE\Crawler\Controller\CrawlerController;
24	use AOE\Crawler\Domain\Model\Reason;
25	use AOE\Crawler\Event\InvokeQueueChangeEvent;
26	use AOE\Crawler\Utility\MessageUtility;
27	use Psr\Http\Message\ResponseInterface;
28	use Psr\Http\Message\ServerRequestInterface;
29	use Symfony\Contracts\Service\Attribute\Required;
30	use TYPO3\CMS\Backend\Routing\Exception\RouteNotFoundException;
31	use TYPO3\CMS\Backend\Routing\UriBuilder;
32	use TYPO3\CMS\Backend\Template\ModuleTemplate;
33	use TYPO3\CMS\Core\EventDispatcher\EventDispatcher;
34	use TYPO3\CMS\Core\Http\Uri;
35	use TYPO3\CMS\Core\Utility\GeneralUtility;
36
37	/**
38	* @internal since v12.0.0
39	*/
40	class BackendModuleStartCrawlingController extends AbstractBackendModuleController implements BackendModuleControllerInterface
41	{
42	private const BACKEND_MODULE = 'web_site_crawler_start';
43	private const LINE_FEED = 10;
44	private const CARRIAGE_RETURN = 13;
45	private int $reqMinute = 1000;
46	private EventDispatcher $eventDispatcher;
47
48	/**
49	* @var array holds the selection of configuration from the configuration selector box
50	*/
51	private $incomingConfigurationSelection = [];
52
53	public function __construct(
54	private readonly CrawlerController $crawlerController,
55	private readonly UriBuilder $backendUriBuilder,
56	) {
57	}	×
58
59	#[Required]
60	public function setEventDispatcher(): void
61	{
62	$this->eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);	×
63	}
64
65	public function handleRequest(ServerRequestInterface $request): ResponseInterface
66	{
67	$this->makeCrawlerProcessableChecks($this->extensionSettings);	×
68	$this->pageUid = (int) ($request->getQueryParams()['id'] ?? -1);	×
69	$this->moduleTemplate = $this->setupView($request, $this->pageUid);	×
70	$this->moduleTemplate->makeDocHeaderModuleMenu([	×
71	'id' => $request->getQueryParams()['id'] ?? -1,	×
72	]);	×
73
74	$this->assignValues($request);	×
75	return $this->moduleTemplate->renderResponse('Backend/ShowCrawlerInformation');	×
76	}
77
78	private function assignValues(ServerRequestInterface $request): ModuleTemplate
79	{
80	$logUrl = $this->backendUriBuilder->buildUriFromRoute('web_site_crawler_log', [	×
81	'id' => $this->pageUid,	×
82	]);	×
83
84	$crawlingDepth = $request->getParsedBody()['crawlingDepth'] ?? $request->getQueryParams()['crawlingDepth'] ?? '0';	×
85	$crawlParameter = $request->getParsedBody()['_crawl'] ?? $request->getQueryParams()['_crawl'] ?? null;	×
86	$downloadParameter = $request->getParsedBody()['_download'] ?? $request->getQueryParams()['_download'] ?? null;	×
87	$scheduledTime = $this->getScheduledTime(	×
88	(string) ($request->getParsedBody()['tstamp'] ?? $request->getQueryParams()['tstamp'] ?? null)	×
89	);	×
90	$submitCrawlUrls = isset($crawlParameter);	×
91	$downloadCrawlUrls = isset($downloadParameter);	×
92
93	$this->incomingConfigurationSelection = $request->getParsedBody()['configurationSelection'] ?? $request->getQueryParams()['configurationSelection'] ?? null;	×
94	$this->incomingConfigurationSelection = is_array(	×
95	$this->incomingConfigurationSelection	×
96	) ? $this->incomingConfigurationSelection : [];	×
97
98	//$this->crawlerController = $this->getCrawlerController();
99	$this->crawlerController->setID = GeneralUtility::md5int(microtime());	×
100
101	$queueRows = '';	×
102	$noConfigurationSelected = empty($this->incomingConfigurationSelection)	×
103	\|\| (count($this->incomingConfigurationSelection) === 1 && empty($this->incomingConfigurationSelection[0]));	×
104	if ($noConfigurationSelected) {	×
105	MessageUtility::addWarningMessage(	×
106	$this->getLanguageService()->sL(	×
107	'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.noConfigSelected'	×
108	)	×
109	);	×
110	} else {
111	if ($submitCrawlUrls) {	×
112	$reason = new Reason();	×
113	$reason->setReason(Reason::REASON_GUI_SUBMIT);	×
114	$reason->setDetailText(	×
115	'The user ' . $GLOBALS['BE_USER']->user['username'] . ' added pages to the crawler queue manually'	×
116	);	×
117	$this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));	×
118	}
119
120	$queueRows = $this->crawlerController->getPageTreeAndUrls(	×
121	$this->pageUid,	×
122	$crawlingDepth,	×
123	$scheduledTime,	×
124	$this->reqMinute,	×
125	$submitCrawlUrls,	×
126	$downloadCrawlUrls,	×
127	// Do not filter any processing instructions
128	[],	×
129	$this->incomingConfigurationSelection	×
130	);	×
131	}
132
133	// Download Urls to crawl:
134	$downloadUrls = $this->crawlerController->downloadUrls;	×
135	if ($downloadCrawlUrls) {	×
136	// Creating output header:
137	header('Content-Type: application/octet-stream');	×
138	header('Content-Disposition: attachment; filename=CrawlerUrls.txt');	×
139
140	// Printing the content of the CSV lines:
141	echo implode(chr(self::CARRIAGE_RETURN) . chr(self::LINE_FEED), $downloadUrls);	×
142	exit;	×
143	}
144
145	return $this->moduleTemplate->assignMultiple([	×
146	'currentPageId' => $this->pageUid,	×
147	'noConfigurationSelected' => $noConfigurationSelected,	×
148	'submitCrawlUrls' => $submitCrawlUrls,	×
149	'amountOfUrls' => count($this->crawlerController->duplicateTrack ?? []),	×
150	'selectors' => $this->generateConfigurationSelectors($this->pageUid, $crawlingDepth, $request),	×
151	'queueRows' => $queueRows,	×
152	'displayActions' => 0,	×
153	'actionUrl' => $this->getActionUrl(),	×
154	'logUrl' => $logUrl,	×
155	]);	×
156	}
157
158	/**
159	* Generates the configuration selectors for compiling URLs:
160	*/
161	private function generateConfigurationSelectors(
162	int $pageId,
163	string $crawlingDepth,
164	ServerRequestInterface $request
165	): array {
166	$selectors = [];	×
167	$selectors['depth'] = $this->selectorBox(	×
168	[	×
169	0 => $this->getLanguageService()->sL(	×
170	'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_0'	×
171	),	×
172	1 => $this->getLanguageService()->sL(	×
173	'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_1'	×
174	),	×
175	2 => $this->getLanguageService()->sL(	×
176	'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_2'	×
177	),	×
178	3 => $this->getLanguageService()->sL(	×
179	'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_3'	×
180	),	×
181	4 => $this->getLanguageService()->sL(	×
182	'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_4'	×
183	),	×
184	99 => $this->getLanguageService()->sL(	×
185	'LLL:EXT:core/Resources/Private/Language/locallang_core.xlf:labels.depth_infi'	×
186	),	×
187	],	×
188	'crawlingDepth',	×
189	$crawlingDepth,	×
190	false	×
191	);	×
192
193	// Configurations
194	$availableConfigurations = $this->crawlerController->getConfigurationsForBranch(	×
195	$pageId,	×
196	(int) $crawlingDepth,	×
197	);	×
198	$selectors['configurations'] = $this->selectorBox(	×
199	empty($availableConfigurations) ? [] : array_combine($availableConfigurations, $availableConfigurations),	×
200	'configurationSelection',	×
201	$this->incomingConfigurationSelection,	×
202	true	×
203	);	×
204
205	// Scheduled time:
206	$selectors['scheduled'] = $this->selectorBox(	×
207	[	×
208	'now' => $this->getLanguageService()->sL(	×
209	'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.time.now'	×
210	),	×
211	'midnight' => $this->getLanguageService()->sL(	×
212	'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.time.midnight'	×
213	),	×
214	'04:00' => $this->getLanguageService()->sL(	×
215	'LLL:EXT:crawler/Resources/Private/Language/locallang.xlf:labels.time.4am'	×
216	),	×
217	],	×
218	'tstamp',	×
219	$request->getParsedBody()['tstamp'] ?? null,	×
220	false	×
221	);	×
222
223	return $selectors;	×
224	}
225
226	/**
227	* Create selector box
228	*
229	* @param array $optArray Options key(value) => label pairs
230	* @param string $name Selector box name
231	* @param string\|array\|null $value Selector box value (array for multiple...)
232	* @param boolean $multiple If set, will draw multiple box.
233	*
234	* @return string HTML select element
235	*/
236	private function selectorBox($optArray, $name, string\|array\|null $value, bool $multiple): string
237	{
238	if (!is_string($value) \|\| !is_array($value)) {	×
239	$value = '';	×
240	}
241
242	$options = [];	×
243	foreach ($optArray as $key => $val) {	×
244	$selected = (!$multiple && !strcmp($value, (string) $key)) \|\| ($multiple && in_array(	×
245	$key,	×
246	(array) $value,	×
247	true	×
248	));	×
249	$options[] = '	×
250	<option value="' . $key . '" ' . ($selected ? ' selected="selected"' : '') . '>' . htmlspecialchars(	×
251	(string) $val,	×
252	ENT_QUOTES \| ENT_HTML5	×
253	) . '</option>';	×
254	}
255
256	return '<select class="form-select" name="' . htmlspecialchars(	×
257	$name . ($multiple ? '[]' : ''),	×
258	ENT_QUOTES \| ENT_HTML5	×
259	) . '"' . ($multiple ? ' multiple' : '') . '>' . implode('', $options) . '</select>';	×
260	}
261
262	private function getScheduledTime(string $time): int
263	{
264	$scheduledTime = match ($time) {	×
265	'midnight' => mktime(0, 0, 0),	×
266	'04:00' => mktime(0, 0, 0) + 4 * 3600,	×
267	default => time(),	×
268	};	×
269
270	if (!$scheduledTime) {	×
271	return time();	×
272	}
273
274	return $scheduledTime;	×
275	}
276
277	/**
278	* @throws RouteNotFoundException
279	*/
280	private function getActionUrl(): Uri
281	{
282	return GeneralUtility::makeInstance(UrlBuilder::class)->getBackendModuleUrl([], self::BACKEND_MODULE);	×
283	}
284	}

tomasnorre / crawler / 11237471329

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous