8096865523

Committed 29 Feb 2024 01:31PM UTC coverage: 89.905% (-0.2%) from 90.144%

Build # 8096865523

Build Type

push

github

Committed by

web-flow

Commit Message

chore: enforce kwarg logging (#7207)

* chore: add logger which eases logging of extras

* chore: start migrating to key value

* fix: import fixes

* tests: temporarily comment out breaking test

* refactor: move to kwarg based logging

* style: fix import order

* chore: implement self-review comments

* test: drop failing test

* chore: fix more import orders

* docs: add changelog

* tests: fix broken tests

* chore: fix getting the frames

* chore: add comment

* chore: cleanup

* chore: adapt remaining `%s` usages

Run Details

5281 of 5874 relevant lines covered (89.9%)

0.9 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

33.33

haystack/utils/requests_utils.py

import logging
from typing import List, Optional

import requests
from tenacity import after_log, before_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential

logger = logging.getLogger(__file__)


def request_with_retry(
    attempts: int = 3, status_codes_to_retry: Optional[List[int]] = None, **kwargs
) -> requests.Response:
    """
    request_with_retry is a simple wrapper function that executes an HTTP request
    with a configurable exponential backoff retry on failures.

    All kwargs will be passed to ``requests.request``, so it accepts the same arguments.

    Example Usage:
    --------------

    # Sending an HTTP request with default retry configs
    res = request_with_retry(method="GET", url="https://example.com")

    # Sending an HTTP request with custom number of attempts
    res = request_with_retry(method="GET", url="https://example.com", attempts=10)

    # Sending an HTTP request with custom HTTP codes to retry
    res = request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=[408, 503])

    # Sending an HTTP request with custom timeout in seconds
    res = request_with_retry(method="GET", url="https://example.com", timeout=5)

    # Sending an HTTP request with custom authorization handling
    class CustomAuth(requests.auth.AuthBase):
        def __call__(self, r):
            r.headers["authorization"] = "Basic <my_token_here>"
            return r

    res = request_with_retry(method="GET", url="https://example.com", auth=CustomAuth())

    # All of the above combined
    res = request_with_retry(
        method="GET",
        url="https://example.com",
        auth=CustomAuth(),
        attempts=10,
        status_codes_to_retry=[408, 503],
        timeout=5
    )

    # Sending a POST request
    res = request_with_retry(method="POST", url="https://example.com", data={"key": "value"}, attempts=10)

    # Retry all 5xx status codes
    res = request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=list(range(500, 600)))

    :param attempts: Maximum number of attempts to retry the request, defaults to 3
    :param status_codes_to_retry: List of HTTP status codes that will trigger a retry, defaults to [408, 418, 429, 503]:
        - `408: Request Timeout`
        - `418`
        - `429: Too Many Requests`
        - `503: Service Unavailable`
    :param **kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    """

    if status_codes_to_retry is None:
        status_codes_to_retry = [408, 418, 429, 503]

    @retry(
        reraise=True,
        wait=wait_exponential(),
        retry=retry_if_exception_type((requests.HTTPError, TimeoutError)),
        stop=stop_after_attempt(attempts),
        before=before_log(logger, logging.DEBUG),
        after=after_log(logger, logging.DEBUG),
    )
    def run():
        timeout = kwargs.pop("timeout", 10)
        res = requests.request(**kwargs, timeout=timeout)

        if res.status_code in status_codes_to_retry:
            # We raise only for the status codes that must trigger a retry
            res.raise_for_status()

        return res

    res = run()
    # We raise here too in case the request failed with a status code that
    # won't trigger a retry, this way the call will still cause an explicit exception
    res.raise_for_status()
    return res

1	import logging	1✔
2	from typing import List, Optional	1✔
3
4	import requests	1✔
5	from tenacity import after_log, before_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential	1✔
6
7	logger = logging.getLogger(__file__)	1✔
8
9
10	def request_with_retry(	1✔
11	attempts: int = 3, status_codes_to_retry: Optional[List[int]] = None, **kwargs
12	) -> requests.Response:
13	"""
14	request_with_retry is a simple wrapper function that executes an HTTP request
15	with a configurable exponential backoff retry on failures.
16
17	All kwargs will be passed to ``requests.request``, so it accepts the same arguments.
18
19	Example Usage:
20	--------------
21
22	# Sending an HTTP request with default retry configs
23	res = request_with_retry(method="GET", url="https://example.com")
24
25	# Sending an HTTP request with custom number of attempts
26	res = request_with_retry(method="GET", url="https://example.com", attempts=10)
27
28	# Sending an HTTP request with custom HTTP codes to retry
29	res = request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=[408, 503])
30
31	# Sending an HTTP request with custom timeout in seconds
32	res = request_with_retry(method="GET", url="https://example.com", timeout=5)
33
34	# Sending an HTTP request with custom authorization handling
35	class CustomAuth(requests.auth.AuthBase):
36	def __call__(self, r):
37	r.headers["authorization"] = "Basic <my_token_here>"
38	return r
39
40	res = request_with_retry(method="GET", url="https://example.com", auth=CustomAuth())
41
42	# All of the above combined
43	res = request_with_retry(
44	method="GET",
45	url="https://example.com",
46	auth=CustomAuth(),
47	attempts=10,
48	status_codes_to_retry=[408, 503],
49	timeout=5
50	)
51
52	# Sending a POST request
53	res = request_with_retry(method="POST", url="https://example.com", data={"key": "value"}, attempts=10)
54
55	# Retry all 5xx status codes
56	res = request_with_retry(method="GET", url="https://example.com", status_codes_to_retry=list(range(500, 600)))
57
58	:param attempts: Maximum number of attempts to retry the request, defaults to 3
59	:param status_codes_to_retry: List of HTTP status codes that will trigger a retry, defaults to [408, 418, 429, 503]:
60	- `408: Request Timeout`
61	- `418`
62	- `429: Too Many Requests`
63	- `503: Service Unavailable`
64	:param **kwargs: Optional arguments that ``request`` takes.
65	:return: :class:`Response <Response>` object
66	"""
67
68	if status_codes_to_retry is None:	×
69	status_codes_to_retry = [408, 418, 429, 503]	×
70
71	@retry(	×
72	reraise=True,
73	wait=wait_exponential(),
74	retry=retry_if_exception_type((requests.HTTPError, TimeoutError)),
75	stop=stop_after_attempt(attempts),
76	before=before_log(logger, logging.DEBUG),
77	after=after_log(logger, logging.DEBUG),
78	)
79	def run():	×
80	timeout = kwargs.pop("timeout", 10)	×
81	res = requests.request(**kwargs, timeout=timeout)	×
82
83	if res.status_code in status_codes_to_retry:	×
84	# We raise only for the status codes that must trigger a retry
85	res.raise_for_status()	×
86
87	return res	×
88
89	res = run()	×
90	# We raise here too in case the request failed with a status code that
91	# won't trigger a retry, this way the call will still cause an explicit exception
92	res.raise_for_status()	×
93	return res	×

deepset-ai / haystack / 8096865523

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous