6181253459

Committed 08 Sep 2023 06:04AM UTC coverage: 94.614% (+0.06%) from 94.556%

Build # 6181253459

Build Type

push

github-actions

Committed by

web-flow

Commit Message

Merge pull request #9159 from stash86/fix-adjust

remove old codes when we only can do partial entries

Run Details

2 of 2 new or added lines in 1 file covered. (100.0%)

19114 of 20202 relevant lines covered (94.61%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.59

/freqtrade/plugins/pairlist/RemotePairList.py

"""
Remote PairList provider

Provides pair list fetched from a remote source
"""
import logging
from pathlib import Path
from typing import Any, Dict, List, Tuple

import rapidjson
import requests
from cachetools import TTLCache

from freqtrade import __version__
from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
from freqtrade.constants import Config
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist


logger = logging.getLogger(__name__)


class RemotePairList(IPairList):

    is_pairlist_generator = True

    def __init__(self, exchange, pairlistmanager,
                 config: Config, pairlistconfig: Dict[str, Any],
                 pairlist_pos: int) -> None:
        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)

        if 'number_assets' not in self._pairlistconfig:
            raise OperationalException(
                '`number_assets` not specified. Please check your configuration '
                'for "pairlist.config.number_assets"')

        if 'pairlist_url' not in self._pairlistconfig:
            raise OperationalException(
                '`pairlist_url` not specified. Please check your configuration '
                'for "pairlist.config.pairlist_url"')

        self._mode = self._pairlistconfig.get('mode', 'whitelist')
        self._processing_mode = self._pairlistconfig.get('processing_mode', 'filter')
        self._number_pairs = self._pairlistconfig['number_assets']
        self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
        self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
        self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
        self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
        self._bearer_token = self._pairlistconfig.get('bearer_token', '')
        self._init_done = False
        self._last_pairlist: List[Any] = list()

        if self._mode not in ['whitelist', 'blacklist']:
            raise OperationalException(
                '`mode` not configured correctly. Supported Modes '
                'are "whitelist","blacklist"')

        if self._processing_mode not in ['filter', 'append']:
            raise OperationalException(
                '`processing_mode` not configured correctly. Supported Modes '
                'are "filter","append"')

        if self._pairlist_pos == 0 and self._mode == 'blacklist':
            raise OperationalException(
                'A `blacklist` mode RemotePairList can not be on the first '
                'position of your pairlist.')

    @property
    def needstickers(self) -> bool:
        """
        Boolean property defining if tickers are necessary.
        If no Pairlist requires tickers, an empty Dict is passed
        as tickers argument to filter_pairlist
        """
        return False

    def short_desc(self) -> str:
        """
        Short whitelist method description - used for startup-messages
        """
        return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."

    @staticmethod
    def description() -> str:
        return "Retrieve pairs from a remote API or local file."

    @staticmethod
    def available_parameters() -> Dict[str, PairlistParameter]:
        return {
            "pairlist_url": {
                "type": "string",
                "default": "",
                "description": "URL to fetch pairlist from",
                "help": "URL to fetch pairlist from",
            },
            "number_assets": {
                "type": "number",
                "default": 30,
                "description": "Number of assets",
                "help": "Number of assets to use from the pairlist.",
            },
            "mode": {
                "type": "option",
                "default": "whitelist",
                "options": ["whitelist", "blacklist"],
                "description": "Pairlist mode",
                "help": "Should this pairlist operate as a whitelist or blacklist?",
            },
            "processing_mode": {
                "type": "option",
                "default": "filter",
                "options": ["filter", "append"],
                "description": "Processing mode",
                "help": "Append pairs to incomming pairlist or filter them?",
            },
            **IPairList.refresh_period_parameter(),
            "keep_pairlist_on_failure": {
                "type": "boolean",
                "default": True,
                "description": "Keep last pairlist on failure",
                "help": "Keep last pairlist on failure",
            },
            "read_timeout": {
                "type": "number",
                "default": 60,
                "description": "Read timeout",
                "help": "Request timeout for remote pairlist",
            },
            "bearer_token": {
                "type": "string",
                "default": "",
                "description": "Bearer token",
                "help": "Bearer token - used for auth against the upstream service.",
            },
        }

    def process_json(self, jsonparse) -> List[str]:

        pairlist = jsonparse.get('pairs', [])
        remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))

        if self._refresh_period < remote_refresh_period:
            self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
                          f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)

            self._refresh_period = remote_refresh_period
            self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)

        self._init_done = True

        return pairlist

    def return_last_pairlist(self) -> List[str]:
        if self._keep_pairlist_on_failure:
            pairlist = self._last_pairlist
            self.log_once('Keeping last fetched pairlist', logger.info)
        else:
            pairlist = []

        return pairlist

    def fetch_pairlist(self) -> Tuple[List[str], float]:

        headers = {
            'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
        }

        if self._bearer_token:
            headers['Authorization'] = f'Bearer {self._bearer_token}'

        try:
            response = requests.get(self._pairlist_url, headers=headers,
                                    timeout=self._read_timeout)
            content_type = response.headers.get('content-type')
            time_elapsed = response.elapsed.total_seconds()

            if "application/json" in str(content_type):
                jsonparse = response.json()

                try:
                    pairlist = self.process_json(jsonparse)
                except Exception as e:

                    if self._init_done:
                        pairlist = self.return_last_pairlist()
                        logger.warning(f'Error while processing JSON data: {type(e)}')
                    else:
                        raise OperationalException(f'Error while processing JSON data: {type(e)}')

            else:
                if self._init_done:
                    self.log_once(f'Error: RemotePairList is not of type JSON: '
                                  f' {self._pairlist_url}', logger.info)
                    pairlist = self.return_last_pairlist()
                else:
                    raise OperationalException('RemotePairList is not of type JSON, abort.')

        except requests.exceptions.RequestException:
            self.log_once(f'Was not able to fetch pairlist from:'
                          f' {self._pairlist_url}', logger.info)

            pairlist = self.return_last_pairlist()

            time_elapsed = 0

        return pairlist, time_elapsed

    def gen_pairlist(self, tickers: Tickers) -> List[str]:
        """
        Generate the pairlist
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: List of pairs
        """

        if self._init_done:
            pairlist = self._pair_cache.get('pairlist')
            if pairlist == [None]:
                # Valid but empty pairlist.
                return []
        else:
            pairlist = []

        time_elapsed = 0.0

        if pairlist:
            # Item found - no refresh necessary
            return pairlist.copy()
        else:
            if self._pairlist_url.startswith("file:///"):
                filename = self._pairlist_url.split("file:///", 1)[1]
                file_path = Path(filename)

                if file_path.exists():
                    with file_path.open() as json_file:
                        # Load the JSON data into a dictionary
                        jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)

                        try:
                            pairlist = self.process_json(jsonparse)
                        except Exception as e:
                            if self._init_done:
                                pairlist = self.return_last_pairlist()
                                logger.warning(f'Error while processing JSON data: {type(e)}')
                            else:
                                raise OperationalException('Error while processing'
                                                           f'JSON data: {type(e)}')
                else:
                    raise ValueError(f"{self._pairlist_url} does not exist.")
            else:
                # Fetch Pairlist from Remote URL
                pairlist, time_elapsed = self.fetch_pairlist()

        self.log_once(f"Fetched pairs: {pairlist}", logger.debug)

        pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys()))
        pairlist = self._whitelist_for_active_markets(pairlist)
        pairlist = pairlist[:self._number_pairs]

        if pairlist:
            self._pair_cache['pairlist'] = pairlist.copy()
        else:
            # If pairlist is empty, set a dummy value to avoid fetching again
            self._pair_cache['pairlist'] = [None]

        if time_elapsed != 0.0:
            self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
        else:
            self.log_once('Fetched Pairlist.', logger.info)

        self._last_pairlist = list(pairlist)

        return pairlist

    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
        Called on each bot iteration - please use internal caching if necessary
        :param pairlist: pairlist to filter or sort
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: new whitelist
        """
        rpl_pairlist = self.gen_pairlist(tickers)
        merged_list = []
        filtered = []

        if self._mode == "whitelist":
            if self._processing_mode == "filter":
                merged_list = [pair for pair in pairlist if pair in rpl_pairlist]
            elif self._processing_mode == "append":
                merged_list = pairlist + rpl_pairlist
            merged_list = sorted(set(merged_list), key=merged_list.index)
        else:
            for pair in pairlist:
                if pair not in rpl_pairlist:
                    merged_list.append(pair)
                else:
                    filtered.append(pair)
            if filtered:
                self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info)

        merged_list = merged_list[:self._number_pairs]
        return merged_list

1	"""
2	Remote PairList provider
3
4	Provides pair list fetched from a remote source
5	"""
6	import logging	1✔
7	from pathlib import Path	1✔
8	from typing import Any, Dict, List, Tuple	1✔
9
10	import rapidjson	1✔
11	import requests	1✔
12	from cachetools import TTLCache	1✔
13
14	from freqtrade import __version__	1✔
15	from freqtrade.configuration.load_config import CONFIG_PARSE_MODE	1✔
16	from freqtrade.constants import Config	1✔
17	from freqtrade.exceptions import OperationalException	1✔
18	from freqtrade.exchange.types import Tickers	1✔
19	from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter	1✔
20	from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist	1✔
21
22
23	logger = logging.getLogger(__name__)	1✔
24
25
26	class RemotePairList(IPairList):	1✔
27
28	is_pairlist_generator = True	1✔
29
30	def __init__(self, exchange, pairlistmanager,	1✔
31	config: Config, pairlistconfig: Dict[str, Any],
32	pairlist_pos: int) -> None:
33	super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)	1✔
34
35	if 'number_assets' not in self._pairlistconfig:	1✔
36	raise OperationalException(	1✔
37	'`number_assets` not specified. Please check your configuration '
38	'for "pairlist.config.number_assets"')
39
40	if 'pairlist_url' not in self._pairlistconfig:	1✔
41	raise OperationalException(	1✔
42	'`pairlist_url` not specified. Please check your configuration '
43	'for "pairlist.config.pairlist_url"')
44
45	self._mode = self._pairlistconfig.get('mode', 'whitelist')	1✔
46	self._processing_mode = self._pairlistconfig.get('processing_mode', 'filter')	1✔
47	self._number_pairs = self._pairlistconfig['number_assets']	1✔
48	self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)	1✔
49	self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)	1✔
50	self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)	1✔
51	self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')	1✔
52	self._read_timeout = self._pairlistconfig.get('read_timeout', 60)	1✔
53	self._bearer_token = self._pairlistconfig.get('bearer_token', '')	1✔
54	self._init_done = False	1✔
55	self._last_pairlist: List[Any] = list()	1✔
56
57	if self._mode not in ['whitelist', 'blacklist']:	1✔
58	raise OperationalException(	1✔
59	'`mode` not configured correctly. Supported Modes '
60	'are "whitelist","blacklist"')
61
62	if self._processing_mode not in ['filter', 'append']:	1✔
63	raise OperationalException(	1✔
64	'`processing_mode` not configured correctly. Supported Modes '
65	'are "filter","append"')
66
67	if self._pairlist_pos == 0 and self._mode == 'blacklist':	1✔
68	raise OperationalException(	1✔
69	'A `blacklist` mode RemotePairList can not be on the first '
70	'position of your pairlist.')
71
72	@property	1✔
73	def needstickers(self) -> bool:	1✔
74	"""
75	Boolean property defining if tickers are necessary.
76	If no Pairlist requires tickers, an empty Dict is passed
77	as tickers argument to filter_pairlist
78	"""
79	return False	1✔
80
81	def short_desc(self) -> str:	1✔
82	"""
83	Short whitelist method description - used for startup-messages
84	"""
85	return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."	1✔
86
87	@staticmethod	1✔
88	def description() -> str:	1✔
89	return "Retrieve pairs from a remote API or local file."	1✔
90
91	@staticmethod	1✔
92	def available_parameters() -> Dict[str, PairlistParameter]:	1✔
93	return {	1✔
94	"pairlist_url": {
95	"type": "string",
96	"default": "",
97	"description": "URL to fetch pairlist from",
98	"help": "URL to fetch pairlist from",
99	},
100	"number_assets": {
101	"type": "number",
102	"default": 30,
103	"description": "Number of assets",
104	"help": "Number of assets to use from the pairlist.",
105	},
106	"mode": {
107	"type": "option",
108	"default": "whitelist",
109	"options": ["whitelist", "blacklist"],
110	"description": "Pairlist mode",
111	"help": "Should this pairlist operate as a whitelist or blacklist?",
112	},
113	"processing_mode": {
114	"type": "option",
115	"default": "filter",
116	"options": ["filter", "append"],
117	"description": "Processing mode",
118	"help": "Append pairs to incomming pairlist or filter them?",
119	},
120	**IPairList.refresh_period_parameter(),
121	"keep_pairlist_on_failure": {
122	"type": "boolean",
123	"default": True,
124	"description": "Keep last pairlist on failure",
125	"help": "Keep last pairlist on failure",
126	},
127	"read_timeout": {
128	"type": "number",
129	"default": 60,
130	"description": "Read timeout",
131	"help": "Request timeout for remote pairlist",
132	},
133	"bearer_token": {
134	"type": "string",
135	"default": "",
136	"description": "Bearer token",
137	"help": "Bearer token - used for auth against the upstream service.",
138	},
139	}
140
141	def process_json(self, jsonparse) -> List[str]:	1✔
142
143	pairlist = jsonparse.get('pairs', [])	1✔
144	remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))	1✔
145
146	if self._refresh_period < remote_refresh_period:	1✔
147	self.log_once(f'Refresh Period has been increased from {self._refresh_period}'	1✔
148	f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
149
150	self._refresh_period = remote_refresh_period	1✔
151	self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)	1✔
152
153	self._init_done = True	1✔
154
155	return pairlist	1✔
156
157	def return_last_pairlist(self) -> List[str]:	1✔
158	if self._keep_pairlist_on_failure:	1✔
159	pairlist = self._last_pairlist	1✔
160	self.log_once('Keeping last fetched pairlist', logger.info)	1✔
161	else:
162	pairlist = []	×
163
164	return pairlist	1✔
165
166	def fetch_pairlist(self) -> Tuple[List[str], float]:	1✔
167
168	headers = {	1✔
169	'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
170	}
171
172	if self._bearer_token:	1✔
173	headers['Authorization'] = f'Bearer {self._bearer_token}'	×
174
175	try:	1✔
176	response = requests.get(self._pairlist_url, headers=headers,	1✔
177	timeout=self._read_timeout)
178	content_type = response.headers.get('content-type')	1✔
179	time_elapsed = response.elapsed.total_seconds()	1✔
180
181	if "application/json" in str(content_type):	1✔
182	jsonparse = response.json()	1✔
183
184	try:	1✔
185	pairlist = self.process_json(jsonparse)	1✔
186	except Exception as e:	×
187
188	if self._init_done:	×
189	pairlist = self.return_last_pairlist()	×
190	logger.warning(f'Error while processing JSON data: {type(e)}')	×
191	else:
192	raise OperationalException(f'Error while processing JSON data: {type(e)}')	×
193
194	else:
195	if self._init_done:	1✔
196	self.log_once(f'Error: RemotePairList is not of type JSON: '	×
197	f' {self._pairlist_url}', logger.info)
198	pairlist = self.return_last_pairlist()	×
199	else:
200	raise OperationalException('RemotePairList is not of type JSON, abort.')	1✔
201
202	except requests.exceptions.RequestException:	1✔
203	self.log_once(f'Was not able to fetch pairlist from:'	1✔
204	f' {self._pairlist_url}', logger.info)
205
206	pairlist = self.return_last_pairlist()	1✔
207
208	time_elapsed = 0	1✔
209
210	return pairlist, time_elapsed	1✔
211
212	def gen_pairlist(self, tickers: Tickers) -> List[str]:	1✔
213	"""
214	Generate the pairlist
215	:param tickers: Tickers (from exchange.get_tickers). May be cached.
216	:return: List of pairs
217	"""
218
219	if self._init_done:	1✔
220	pairlist = self._pair_cache.get('pairlist')	1✔
221	if pairlist == [None]:	1✔
222	# Valid but empty pairlist.
223	return []	×
224	else:
225	pairlist = []	1✔
226
227	time_elapsed = 0.0	1✔
228
229	if pairlist:	1✔
230	# Item found - no refresh necessary
231	return pairlist.copy()	×
232	else:
233	if self._pairlist_url.startswith("file:///"):	1✔
234	filename = self._pairlist_url.split("file:///", 1)[1]	1✔
235	file_path = Path(filename)	1✔
236
237	if file_path.exists():	1✔
238	with file_path.open() as json_file:	1✔
239	# Load the JSON data into a dictionary
240	jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)	1✔
241
242	try:	1✔
243	pairlist = self.process_json(jsonparse)	1✔
244	except Exception as e:	×
245	if self._init_done:	×
246	pairlist = self.return_last_pairlist()	×
247	logger.warning(f'Error while processing JSON data: {type(e)}')	×
248	else:
249	raise OperationalException('Error while processing'	×
250	f'JSON data: {type(e)}')
251	else:
252	raise ValueError(f"{self._pairlist_url} does not exist.")	×
253	else:
254	# Fetch Pairlist from Remote URL
255	pairlist, time_elapsed = self.fetch_pairlist()	1✔
256
257	self.log_once(f"Fetched pairs: {pairlist}", logger.debug)	1✔
258
259	pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys()))	1✔
260	pairlist = self._whitelist_for_active_markets(pairlist)	1✔
261	pairlist = pairlist[:self._number_pairs]	1✔
262
263	if pairlist:	1✔
264	self._pair_cache['pairlist'] = pairlist.copy()	1✔
265	else:
266	# If pairlist is empty, set a dummy value to avoid fetching again
267	self._pair_cache['pairlist'] = [None]	×
268
269	if time_elapsed != 0.0:	1✔
270	self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)	1✔
271	else:
272	self.log_once('Fetched Pairlist.', logger.info)	1✔
273
274	self._last_pairlist = list(pairlist)	1✔
275
276	return pairlist	1✔
277
278	def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:	1✔
279	"""
280	Filters and sorts pairlist and returns the whitelist again.
281	Called on each bot iteration - please use internal caching if necessary
282	:param pairlist: pairlist to filter or sort
283	:param tickers: Tickers (from exchange.get_tickers). May be cached.
284	:return: new whitelist
285	"""
286	rpl_pairlist = self.gen_pairlist(tickers)	1✔
287	merged_list = []	1✔
288	filtered = []	1✔
289
290	if self._mode == "whitelist":	1✔
291	if self._processing_mode == "filter":	1✔
292	merged_list = [pair for pair in pairlist if pair in rpl_pairlist]	1✔
293	elif self._processing_mode == "append":	1✔
294	merged_list = pairlist + rpl_pairlist	1✔
295	merged_list = sorted(set(merged_list), key=merged_list.index)	1✔
296	else:
297	for pair in pairlist:	1✔
298	if pair not in rpl_pairlist:	1✔
299	merged_list.append(pair)	1✔
300	else:
301	filtered.append(pair)	1✔
302	if filtered:	1✔
303	self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info)	1✔
304
305	merged_list = merged_list[:self._number_pairs]	1✔
306	return merged_list	1✔

freqtrade / freqtrade / 6181253459

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous