17436872817

Committed 03 Sep 2025 02:34PM UTC coverage: 68.647% (+0.05%) from 68.597%

Build # 17436872817

Build Type

push

github

Committed by

tomasnorre

Commit Message

Revert "[BUGFIX] Prevent warning when guzzle crawl requests return errors (#1167)"

This reverts commit faa0d89cd.

Run Details

8 of 11 new or added lines in 2 files covered. (72.73%)

1872 of 2727 relevant lines covered (68.65%)

3.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.23

/Classes/CrawlStrategy/GuzzleExecutionStrategy.php

<?php

declare(strict_types=1);

namespace AOE\Crawler\CrawlStrategy;

/*
 * (c) 2020 AOE GmbH <dev@aoe.com>
 *
 * This file is part of the TYPO3 Crawler Extension.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use GuzzleHttp\Exception\ConnectException;
use GuzzleHttp\Exception\RequestException;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use TYPO3\CMS\Core\Http\Client\GuzzleClientFactory;
use TYPO3\CMS\Core\Http\RequestFactory;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * Calls Guzzle / CURL (based on TYPO3 settings) for fetching a URL.
 * @internal since v12.0.0
 */
class GuzzleExecutionStrategy implements LoggerAwareInterface, CrawlStrategyInterface
{
    use LoggerAwareTrait;

    /**
     * Sets up a CURL / Guzzle Request for fetching the request.
     *
     * @return bool|mixed
     */
    public function fetchUrlContents(UriInterface $url, string $crawlerId)
    {
        $reqHeaders = $this->buildRequestHeaders($crawlerId);

        $options = [
            'headers' => $reqHeaders,
            'connect_timeout' => 5.0,
        ];
        if ($url->getUserInfo()) {
            $options['auth'] = explode(':', $url->getUserInfo());
        }
        try {
            $url = (string) $url;
            $response = $this->getResponse($url, $options);
            return unserialize($response->getHeaderLine('X-T3Crawler-Meta'));
        } catch (RequestException $e) {
            $response = $e->getResponse();
            $message = ($response ? $response->getStatusCode() : 0)
                . chr(32)
                . ($response ? $response->getReasonPhrase() : $e->getMessage());

            $this->logger->debug(
                sprintf('Error while opening "%s" - ' . $message, $url),
                [
                    'crawlerId' => $crawlerId,
                ]
            );
            return $message;
        } catch (ConnectException $e) {
            $message = $e->getCode() . chr(32) . $e->getMessage();

            $this->logger->debug(
                sprintf('Error while opening "%s" - ' . $message, $url),
                [
                    'crawlerId' => $crawlerId,
                ]
            );
            return $message;
        }
    }

    protected function getResponse(string $url, array $options): ResponseInterface
    {
        $guzzleClientFactory = GeneralUtility::makeInstance(GuzzleClientFactory::class);
        return GeneralUtility::makeInstance(RequestFactory::class, $guzzleClientFactory)
            ->request($url, 'GET', $options);
    }

    /**
     * Builds HTTP request headers.
     */
    private function buildRequestHeaders(string $crawlerId): array
    {
        return [
            'Connection' => 'close',
            'X-T3Crawler' => $crawlerId,
            'User-Agent' => 'TYPO3 crawler',
        ];
    }
}

1	<?php
2
3	declare(strict_types=1);
4
5	namespace AOE\Crawler\CrawlStrategy;
6
7	/*
8	* (c) 2020 AOE GmbH <dev@aoe.com>
9	*
10	* This file is part of the TYPO3 Crawler Extension.
11	*
12	* It is free software; you can redistribute it and/or modify it under
13	* the terms of the GNU General Public License, either version 2
14	* of the License, or any later version.
15	*
16	* For the full copyright and license information, please read the
17	* LICENSE.txt file that was distributed with this source code.
18	*
19	* The TYPO3 project - inspiring people to share!
20	*/
21
22	use GuzzleHttp\Exception\ConnectException;
23	use GuzzleHttp\Exception\RequestException;
24	use Psr\Http\Message\ResponseInterface;
25	use Psr\Http\Message\UriInterface;
26	use Psr\Log\LoggerAwareInterface;
27	use Psr\Log\LoggerAwareTrait;
28	use TYPO3\CMS\Core\Http\Client\GuzzleClientFactory;
29	use TYPO3\CMS\Core\Http\RequestFactory;
30	use TYPO3\CMS\Core\Utility\GeneralUtility;
31
32	/**
33	* Calls Guzzle / CURL (based on TYPO3 settings) for fetching a URL.
34	* @internal since v12.0.0
35	*/
36	class GuzzleExecutionStrategy implements LoggerAwareInterface, CrawlStrategyInterface
37	{
38	use LoggerAwareTrait;
39
40	/**
41	* Sets up a CURL / Guzzle Request for fetching the request.
42	*
43	* @return bool\|mixed
44	*/
45	public function fetchUrlContents(UriInterface $url, string $crawlerId)
46	{
47	$reqHeaders = $this->buildRequestHeaders($crawlerId);	3✔
48
49	$options = [	3✔
50	'headers' => $reqHeaders,	3✔
51	'connect_timeout' => 5.0,	3✔
52	];	3✔
53	if ($url->getUserInfo()) {	3✔
54	$options['auth'] = explode(':', $url->getUserInfo());	×
55	}
56	try {
57	$url = (string) $url;	3✔
58	$response = $this->getResponse($url, $options);	3✔
59	return unserialize($response->getHeaderLine('X-T3Crawler-Meta'));	2✔
60	} catch (RequestException $e) {	1✔
61	$response = $e->getResponse();	×
62	$message = ($response ? $response->getStatusCode() : 0)	×
63	. chr(32)	×
64	. ($response ? $response->getReasonPhrase() : $e->getMessage());	×
65
66	$this->logger->debug(	×
67	sprintf('Error while opening "%s" - ' . $message, $url),	×
68	[	×
69	'crawlerId' => $crawlerId,	×
70	]	×
71	);	×
NEW 72	return $message;	×
73	} catch (ConnectException $e) {	1✔
74	$message = $e->getCode() . chr(32) . $e->getMessage();	1✔
75
76	$this->logger->debug(	1✔
77	sprintf('Error while opening "%s" - ' . $message, $url),	1✔
78	[	1✔
79	'crawlerId' => $crawlerId,	1✔
80	]	1✔
81	);	1✔
82	return $message;	1✔
83	}
84	}
85
86	protected function getResponse(string $url, array $options): ResponseInterface
87	{
88	$guzzleClientFactory = GeneralUtility::makeInstance(GuzzleClientFactory::class);	2✔
89	return GeneralUtility::makeInstance(RequestFactory::class, $guzzleClientFactory)	2✔
90	->request($url, 'GET', $options);	2✔
91	}
92
93	/**
94	* Builds HTTP request headers.
95	*/
96	private function buildRequestHeaders(string $crawlerId): array
97	{
98	return [	3✔
99	'Connection' => 'close',	3✔
100	'X-T3Crawler' => $crawlerId,	3✔
101	'User-Agent' => 'TYPO3 crawler',	3✔
102	];	3✔
103	}
104	}

tomasnorre / crawler / 17436872817

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous