11508304044

Committed 24 Oct 2024 10:16PM UTC coverage: 100.0%. Remained the same

Build # 11508304044

Build Type

Pull #33

github

Committed by

ape364

Commit Message

fix: pragma: no cover

Pull Request Pull Request #33: feat: add api keys rotating

Run Details

41 of 41 new or added lines in 4 files covered. (100.0%)

699 of 699 relevant lines covered (100.0%)

4.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0

/aioetherscan/modules/extra/generators/blocks_parser.py

import logging
from typing import AsyncIterator, Any, Iterable
from typing import Callable

from aioetherscan.exceptions import EtherscanClientApiError
from aioetherscan.modules.extra.generators.blocks_range import BlocksRange
from aioetherscan.modules.extra.generators.helpers import get_max_block_number, drop_block

Transfer = dict[str, Any]


class BlocksParser:
    _OFFSET: int = 10_000

    def __init__(
        self,
        api_method: Callable,
        request_params: dict[str, Any],
        start_block: int,
        end_block: int,
        blocks_limit: int,
        blocks_limit_divider: int,
    ) -> None:
        self._api_method = api_method
        self._request_params = request_params

        self._blocks_range = BlocksRange(start_block, end_block, blocks_limit, blocks_limit_divider)

        self._logger = logging.getLogger(__name__)
        self._total_txs = 0

    async def txs_generator(self) -> AsyncIterator[Transfer]:
        while self._blocks_range.blocks_left:
            try:
                blocks_range = self._blocks_range.get_blocks_range()
                last_seen_block, transfers = await self._fetch_blocks_range(blocks_range)
            except EtherscanClientApiError as e:
                self._logger.error(f'Error: {e}')
                self._blocks_range.limit.reduce()
            else:
                self._blocks_range.current_block = last_seen_block + 1
                self._blocks_range.limit.restore()

                for transfer in transfers:
                    yield transfer

                self._logger.info(
                    f'[{self._blocks_range.blocks_done / self._blocks_range.size:.2%}] '
                    f'Current block {self._blocks_range.current_block:,} '
                    f'({self._blocks_range.blocks_left:,} blocks left)'
                )

    def _make_request_params(self, blocks_range: range) -> Transfer:
        current_params = dict(
            start_block=blocks_range.start,
            end_block=blocks_range.stop,
            page=1,
            offset=self._OFFSET,
        )
        params = self._request_params | current_params
        self._logger.debug(f'Request params: {params}')
        return params

    async def _fetch_blocks_range(self, blocks_range: range) -> tuple[int, Iterable[Transfer]]:
        try:
            request_params = self._make_request_params(blocks_range)
            transfers = await self._api_method(**request_params)
        except EtherscanClientApiError as e:
            if e.message == 'No transactions found':
                return blocks_range.stop, []
            raise
        else:
            if not transfers:
                self._logger.debug('No transfers found')
                return blocks_range.stop, []

            transfers_count = len(transfers)
            self._total_txs += transfers_count
            self._logger.debug(f'Got {transfers_count:,} transfers, {self._total_txs:,} total')

            transfers_max_block = get_max_block_number(transfers)

            if transfers_count == self._OFFSET:
                self._logger.debug(
                    f'Probably not all txs have been fetched, dropping txs with the last block {transfers_max_block:,}'
                )
                return transfers_max_block - 1, drop_block(transfers, transfers_max_block)
            else:
                self._logger.debug('All txs have been fetched')
                return transfers_max_block, transfers

1	import logging	4✔
2	from typing import AsyncIterator, Any, Iterable	4✔
3	from typing import Callable	4✔
4
5	from aioetherscan.exceptions import EtherscanClientApiError	4✔
6	from aioetherscan.modules.extra.generators.blocks_range import BlocksRange	4✔
7	from aioetherscan.modules.extra.generators.helpers import get_max_block_number, drop_block	4✔
8
9	Transfer = dict[str, Any]	4✔
10
11
12	class BlocksParser:	4✔
13	_OFFSET: int = 10_000	4✔
14
15	def __init__(	4✔
16	self,
17	api_method: Callable,
18	request_params: dict[str, Any],
19	start_block: int,
20	end_block: int,
21	blocks_limit: int,
22	blocks_limit_divider: int,
23	) -> None:
24	self._api_method = api_method	4✔
25	self._request_params = request_params	4✔
26
27	self._blocks_range = BlocksRange(start_block, end_block, blocks_limit, blocks_limit_divider)	4✔
28
29	self._logger = logging.getLogger(__name__)	4✔
30	self._total_txs = 0	4✔
31
32	async def txs_generator(self) -> AsyncIterator[Transfer]:	4✔
33	while self._blocks_range.blocks_left:	4✔
34	try:	4✔
35	blocks_range = self._blocks_range.get_blocks_range()	4✔
36	last_seen_block, transfers = await self._fetch_blocks_range(blocks_range)	4✔
37	except EtherscanClientApiError as e:	4✔
38	self._logger.error(f'Error: {e}')	4✔
39	self._blocks_range.limit.reduce()	4✔
40	else:
41	self._blocks_range.current_block = last_seen_block + 1	4✔
42	self._blocks_range.limit.restore()	4✔
43
44	for transfer in transfers:	4✔
45	yield transfer	4✔
46
47	self._logger.info(	4✔
48	f'[{self._blocks_range.blocks_done / self._blocks_range.size:.2%}] '
49	f'Current block {self._blocks_range.current_block:,} '
50	f'({self._blocks_range.blocks_left:,} blocks left)'
51	)
52
53	def _make_request_params(self, blocks_range: range) -> Transfer:	4✔
54	current_params = dict(	4✔
55	start_block=blocks_range.start,
56	end_block=blocks_range.stop,
57	page=1,
58	offset=self._OFFSET,
59	)
60	params = self._request_params \| current_params	4✔
61	self._logger.debug(f'Request params: {params}')	4✔
62	return params	4✔
63
64	async def _fetch_blocks_range(self, blocks_range: range) -> tuple[int, Iterable[Transfer]]:	4✔
65	try:	4✔
66	request_params = self._make_request_params(blocks_range)	4✔
67	transfers = await self._api_method(**request_params)	4✔
68	except EtherscanClientApiError as e:	4✔
69	if e.message == 'No transactions found':	4✔
70	return blocks_range.stop, []	4✔
71	raise	4✔
72	else:
73	if not transfers:	4✔
74	self._logger.debug('No transfers found')	4✔
75	return blocks_range.stop, []	4✔
76
77	transfers_count = len(transfers)	4✔
78	self._total_txs += transfers_count	4✔
79	self._logger.debug(f'Got {transfers_count:,} transfers, {self._total_txs:,} total')	4✔
80
81	transfers_max_block = get_max_block_number(transfers)	4✔
82
83	if transfers_count == self._OFFSET:	4✔
84	self._logger.debug(	4✔
85	f'Probably not all txs have been fetched, dropping txs with the last block {transfers_max_block:,}'
86	)
87	return transfers_max_block - 1, drop_block(transfers, transfers_max_block)	4✔
88	else:
89	self._logger.debug('All txs have been fetched')	4✔
90	return transfers_max_block, transfers	4✔

ape364 / aioetherscan / 11508304044

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous