679e0ec2-f562-4461-97ff-d30f9592cc72

Committed 20 Apr 2024 12:06PM UTC coverage: 80.384% (+0.9%) from 79.454%

Build # 679e0ec2-f562-4461-97ff-d30f9592cc72

Build Type

push

circleci

Committed by

web-flow

Commit Message

Merge pull request #507 from DemocracyClub/parquet-elections-endpoint

Initial elections for postcode endpoint

Run Details

134 of 150 new or added lines in 5 files covered. (89.33%)

1172 of 1458 relevant lines covered (80.38%)

0.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.18

/api/common/async_requests.py

import asyncio
from json import JSONDecodeError
from typing import Dict

import httpx

client = httpx.AsyncClient(http2=True)


class UpstreamApiError(Exception):
    def __init__(self, response_dict: httpx.Response):
        try:
            self.message = {"error": response_dict.json().get("detail", "")}
        except JSONDecodeError:
            self.message = ""
        self.status = response_dict.status_code


async def get_url(key, url_data, request_urls, raise_errors=True):
    response: httpx.Response = await client.get(
        url=url_data["url"],
        params=url_data.get("params", {}),
        headers=url_data.get("headers", {}),
    )
    request_urls[key]["response"] = response
    if raise_errors:
        if request_urls[key]["response"].status_code >= 400:
            raise UpstreamApiError(request_urls[key]["response"])
        request_urls[key]["response"].raise_for_status()


async def async_get_urls(
    requst_urls, raise_errors=True
) -> Dict[str, httpx.Response]:
    await asyncio.gather(
        *[
            get_url(
                key, requst_urls[key], requst_urls, raise_errors=raise_errors
            )
            for key in requst_urls
        ]
    )
    if raise_errors:
        for url, result in requst_urls.items():
            if result["response"].status_code >= 400:
                raise UpstreamApiError(result)
    return requst_urls


class AsyncRequester:
    """
    Used HTTPX and async to request URLs in parallel

    Pass in a dict with the following structure:

    {
        "key1": {
            "url": "https://example.com/url1/",
            "params": {},
            "headers": {}
        },
        "key2": {
            "url": "https://example.com/url2/",
            "params": {},
            "headers": {}
        }
    }

    An async request for each URL will be started, and when they're all complete
    the dict will be returned with `response` objects:

    {
        "key1": {
            "url": "https://example.com/url1/",
            "params": {},
            "headers": {},
            "response": <httpx response object>
        },
        "key2": {
            "url": "https://example.com/url2/",
            "params": {},
            "headers": {},
            "response": <httpx response object>
        }
    }

    """

    USER_AGENT = "devs.DC API"

    def __init__(self, request_dict: Dict):
        self.request_dict = request_dict

    @property
    def get_default_headers(self):
        return {"Accept": "application/json", "User-Agent": self.USER_AGENT}

    def add_default_headers(self):
        for key, value in self.request_dict.items():
            headers = value.get("headers", {})
            headers.update(self.get_default_headers)

    async def get_urls(self, raise_errors=True) -> dict:
        return await async_get_urls(
            self.request_dict, raise_errors=raise_errors
        )

1	import asyncio	1✔
2	from json import JSONDecodeError	1✔
3	from typing import Dict	1✔
4
5	import httpx	1✔
6
7	client = httpx.AsyncClient(http2=True)	1✔
8
9
10	class UpstreamApiError(Exception):	1✔
11	def __init__(self, response_dict: httpx.Response):	1✔
12	try:	1✔
13	self.message = {"error": response_dict.json().get("detail", "")}	1✔
14	except JSONDecodeError:	1✔
15	self.message = ""	1✔
16	self.status = response_dict.status_code	1✔
17
18
19	async def get_url(key, url_data, request_urls, raise_errors=True):	1✔
20	response: httpx.Response = await client.get(	1✔
21	url=url_data["url"],
22	params=url_data.get("params", {}),
23	headers=url_data.get("headers", {}),
24	)
25	request_urls[key]["response"] = response	1✔
26	if raise_errors:	1✔
27	if request_urls[key]["response"].status_code >= 400:	1✔
28	raise UpstreamApiError(request_urls[key]["response"])	1✔
29	request_urls[key]["response"].raise_for_status()	1✔
30
31
32	async def async_get_urls(	1✔
33	requst_urls, raise_errors=True
34	) -> Dict[str, httpx.Response]:
35	await asyncio.gather(	1✔
36	*[
37	get_url(
38	key, requst_urls[key], requst_urls, raise_errors=raise_errors
39	)
40	for key in requst_urls
41	]
42	)
43	if raise_errors:	1✔
44	for url, result in requst_urls.items():	1✔
45	if result["response"].status_code >= 400:	1✔
NEW 46	raise UpstreamApiError(result)	×
47	return requst_urls	1✔
48
49
50	class AsyncRequester:	1✔
51	"""
52	Used HTTPX and async to request URLs in parallel
53
54	Pass in a dict with the following structure:
55
56	{
57	"key1": {
58	"url": "https://example.com/url1/",
59	"params": {},
60	"headers": {}
61	},
62	"key2": {
63	"url": "https://example.com/url2/",
64	"params": {},
65	"headers": {}
66	}
67	}
68
69	An async request for each URL will be started, and when they're all complete
70	the dict will be returned with `response` objects:
71
72	{
73	"key1": {
74	"url": "https://example.com/url1/",
75	"params": {},
76	"headers": {},
77	"response": <httpx response object>
78	},
79	"key2": {
80	"url": "https://example.com/url2/",
81	"params": {},
82	"headers": {},
83	"response": <httpx response object>
84	}
85	}
86
87	"""
88
89	USER_AGENT = "devs.DC API"	1✔
90
91	def __init__(self, request_dict: Dict):	1✔
92	self.request_dict = request_dict	1✔
93
94	@property	1✔
95	def get_default_headers(self):	1✔
96	return {"Accept": "application/json", "User-Agent": self.USER_AGENT}	×
97
98	def add_default_headers(self):	1✔
99	for key, value in self.request_dict.items():	×
100	headers = value.get("headers", {})	×
101	headers.update(self.get_default_headers)	×
102
103	async def get_urls(self, raise_errors=True) -> dict:	1✔
104	return await async_get_urls(	1✔
105	self.request_dict, raise_errors=raise_errors
106	)

DemocracyClub / aggregator-api / 679e0ec2-f562-4461-97ff-d30f9592cc72

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous