9394559170

Committed 26 Apr 2024 06:36AM UTC coverage: 94.656% (-0.02%) from 94.674%

Build # 9394559170

Build Type

push

github

Committed by

xmatthias

Commit Message

Loader should be passed as kwarg for clarity

Run Details

20280 of 21425 relevant lines covered (94.66%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.5

/freqtrade/plugins/pairlist/RemotePairList.py

"""
Remote PairList provider

Provides pair list fetched from a remote source
"""
import logging
from pathlib import Path
from typing import Any, Dict, List, Tuple

import rapidjson
import requests
from cachetools import TTLCache

from freqtrade import __version__
from freqtrade.configuration.load_config import CONFIG_PARSE_MODE
from freqtrade.constants import Config
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist


logger = logging.getLogger(__name__)


class RemotePairList(IPairList):

    is_pairlist_generator = True

    def __init__(self, exchange, pairlistmanager,
                 config: Config, pairlistconfig: Dict[str, Any],
                 pairlist_pos: int) -> None:
        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)

        if 'number_assets' not in self._pairlistconfig:
            raise OperationalException(
                '`number_assets` not specified. Please check your configuration '
                'for "pairlist.config.number_assets"')

        if 'pairlist_url' not in self._pairlistconfig:
            raise OperationalException(
                '`pairlist_url` not specified. Please check your configuration '
                'for "pairlist.config.pairlist_url"')

        self._mode = self._pairlistconfig.get('mode', 'whitelist')
        self._processing_mode = self._pairlistconfig.get('processing_mode', 'filter')
        self._number_pairs = self._pairlistconfig['number_assets']
        self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)
        self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)
        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
        self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')
        self._read_timeout = self._pairlistconfig.get('read_timeout', 60)
        self._bearer_token = self._pairlistconfig.get('bearer_token', '')
        self._init_done = False
        self._save_to_file = self._pairlistconfig.get('save_to_file', None)
        self._last_pairlist: List[Any] = list()

        if self._mode not in ['whitelist', 'blacklist']:
            raise OperationalException(
                '`mode` not configured correctly. Supported Modes '
                'are "whitelist","blacklist"')

        if self._processing_mode not in ['filter', 'append']:
            raise OperationalException(
                '`processing_mode` not configured correctly. Supported Modes '
                'are "filter","append"')

        if self._pairlist_pos == 0 and self._mode == 'blacklist':
            raise OperationalException(
                'A `blacklist` mode RemotePairList can not be on the first '
                'position of your pairlist.')

    @property
    def needstickers(self) -> bool:
        """
        Boolean property defining if tickers are necessary.
        If no Pairlist requires tickers, an empty Dict is passed
        as tickers argument to filter_pairlist
        """
        return False

    def short_desc(self) -> str:
        """
        Short whitelist method description - used for startup-messages
        """
        return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."

    @staticmethod
    def description() -> str:
        return "Retrieve pairs from a remote API or local file."

    @staticmethod
    def available_parameters() -> Dict[str, PairlistParameter]:
        return {
            "pairlist_url": {
                "type": "string",
                "default": "",
                "description": "URL to fetch pairlist from",
                "help": "URL to fetch pairlist from",
            },
            "number_assets": {
                "type": "number",
                "default": 30,
                "description": "Number of assets",
                "help": "Number of assets to use from the pairlist.",
            },
            "mode": {
                "type": "option",
                "default": "whitelist",
                "options": ["whitelist", "blacklist"],
                "description": "Pairlist mode",
                "help": "Should this pairlist operate as a whitelist or blacklist?",
            },
            "processing_mode": {
                "type": "option",
                "default": "filter",
                "options": ["filter", "append"],
                "description": "Processing mode",
                "help": "Append pairs to incoming pairlist or filter them?",
            },
            **IPairList.refresh_period_parameter(),
            "keep_pairlist_on_failure": {
                "type": "boolean",
                "default": True,
                "description": "Keep last pairlist on failure",
                "help": "Keep last pairlist on failure",
            },
            "read_timeout": {
                "type": "number",
                "default": 60,
                "description": "Read timeout",
                "help": "Request timeout for remote pairlist",
            },
            "bearer_token": {
                "type": "string",
                "default": "",
                "description": "Bearer token",
                "help": "Bearer token - used for auth against the upstream service.",
            },
            "save_to_file": {
                "type": "string",
                "default": "",
                "description": "Filename to save processed pairlist to.",
                "help": "Specify a filename to save the processed pairlist in JSON format.",
            },
        }

    def process_json(self, jsonparse) -> List[str]:

        pairlist = jsonparse.get('pairs', [])
        remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))

        if self._refresh_period < remote_refresh_period:
            self.log_once(f'Refresh Period has been increased from {self._refresh_period}'
                          f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)

            self._refresh_period = remote_refresh_period
            self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)

        self._init_done = True

        return pairlist

    def return_last_pairlist(self) -> List[str]:
        if self._keep_pairlist_on_failure:
            pairlist = self._last_pairlist
            self.log_once('Keeping last fetched pairlist', logger.info)
        else:
            pairlist = []

        return pairlist

    def fetch_pairlist(self) -> Tuple[List[str], float]:

        headers = {
            'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
        }

        if self._bearer_token:
            headers['Authorization'] = f'Bearer {self._bearer_token}'

        try:
            response = requests.get(self._pairlist_url, headers=headers,
                                    timeout=self._read_timeout)
            content_type = response.headers.get('content-type')
            time_elapsed = response.elapsed.total_seconds()

            if "application/json" in str(content_type):
                jsonparse = response.json()

                try:
                    pairlist = self.process_json(jsonparse)
                except Exception as e:
                    pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}')
            else:
                pairlist = self._handle_error(f'RemotePairList is not of type JSON.'
                                              f' {self._pairlist_url}')

        except requests.exceptions.RequestException:
            pairlist = self._handle_error(f'Was not able to fetch pairlist from:'
                                          f' {self._pairlist_url}')

            time_elapsed = 0

        return pairlist, time_elapsed

    def _handle_error(self, error: str) -> List[str]:
        if self._init_done:
            self.log_once("Error: " + error, logger.info)
            return self.return_last_pairlist()
        else:
            raise OperationalException(error)

    def gen_pairlist(self, tickers: Tickers) -> List[str]:
        """
        Generate the pairlist
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: List of pairs
        """

        if self._init_done:
            pairlist = self._pair_cache.get('pairlist')
            if pairlist == [None]:
                # Valid but empty pairlist.
                return []
        else:
            pairlist = []

        time_elapsed = 0.0

        if pairlist:
            # Item found - no refresh necessary
            return pairlist.copy()
        else:
            if self._pairlist_url.startswith("file:///"):
                filename = self._pairlist_url.split("file:///", 1)[1]
                file_path = Path(filename)

                if file_path.exists():
                    with file_path.open() as json_file:
                        try:
                            # Load the JSON data into a dictionary
                            jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)
                            pairlist = self.process_json(jsonparse)
                        except Exception as e:
                            pairlist = self._handle_error(f'processing JSON data: {type(e)}')
                else:
                    pairlist = self._handle_error(f"{self._pairlist_url} does not exist.")

            else:
                # Fetch Pairlist from Remote URL
                pairlist, time_elapsed = self.fetch_pairlist()

        self.log_once(f"Fetched pairs: {pairlist}", logger.debug)

        pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys()))
        pairlist = self._whitelist_for_active_markets(pairlist)
        pairlist = pairlist[:self._number_pairs]

        if pairlist:
            self._pair_cache['pairlist'] = pairlist.copy()
        else:
            # If pairlist is empty, set a dummy value to avoid fetching again
            self._pair_cache['pairlist'] = [None]

        if time_elapsed != 0.0:
            self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)
        else:
            self.log_once('Fetched Pairlist.', logger.info)

        self._last_pairlist = list(pairlist)

        if self._save_to_file:
            self.save_pairlist(pairlist, self._save_to_file)

        return pairlist

    def save_pairlist(self, pairlist: List[str], filename: str) -> None:
        pairlist_data = {
            "pairs": pairlist
        }
        try:
            file_path = Path(filename)
            with file_path.open('w') as json_file:
                rapidjson.dump(pairlist_data, json_file)
                logger.info(f"Processed pairlist saved to {filename}")
        except Exception as e:
            logger.error(f"Error saving processed pairlist to {filename}: {e}")

    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
        Called on each bot iteration - please use internal caching if necessary
        :param pairlist: pairlist to filter or sort
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: new whitelist
        """
        rpl_pairlist = self.gen_pairlist(tickers)
        merged_list = []
        filtered = []

        if self._mode == "whitelist":
            if self._processing_mode == "filter":
                merged_list = [pair for pair in pairlist if pair in rpl_pairlist]
            elif self._processing_mode == "append":
                merged_list = pairlist + rpl_pairlist
            merged_list = sorted(set(merged_list), key=merged_list.index)
        else:
            for pair in pairlist:
                if pair not in rpl_pairlist:
                    merged_list.append(pair)
                else:
                    filtered.append(pair)
            if filtered:
                self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info)

        merged_list = merged_list[:self._number_pairs]
        return merged_list

1	"""
2	Remote PairList provider
3
4	Provides pair list fetched from a remote source
5	"""
6	import logging	1✔
7	from pathlib import Path	1✔
8	from typing import Any, Dict, List, Tuple	1✔
9
10	import rapidjson	1✔
11	import requests	1✔
12	from cachetools import TTLCache	1✔
13
14	from freqtrade import __version__	1✔
15	from freqtrade.configuration.load_config import CONFIG_PARSE_MODE	1✔
16	from freqtrade.constants import Config	1✔
17	from freqtrade.exceptions import OperationalException	1✔
18	from freqtrade.exchange.types import Tickers	1✔
19	from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter	1✔
20	from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist	1✔
21
22
23	logger = logging.getLogger(__name__)	1✔
24
25
26	class RemotePairList(IPairList):	1✔
27
28	is_pairlist_generator = True	1✔
29
30	def __init__(self, exchange, pairlistmanager,	1✔
31	config: Config, pairlistconfig: Dict[str, Any],
32	pairlist_pos: int) -> None:
33	super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)	1✔
34
35	if 'number_assets' not in self._pairlistconfig:	1✔
36	raise OperationalException(	1✔
37	'`number_assets` not specified. Please check your configuration '
38	'for "pairlist.config.number_assets"')
39
40	if 'pairlist_url' not in self._pairlistconfig:	1✔
41	raise OperationalException(	1✔
42	'`pairlist_url` not specified. Please check your configuration '
43	'for "pairlist.config.pairlist_url"')
44
45	self._mode = self._pairlistconfig.get('mode', 'whitelist')	1✔
46	self._processing_mode = self._pairlistconfig.get('processing_mode', 'filter')	1✔
47	self._number_pairs = self._pairlistconfig['number_assets']	1✔
48	self._refresh_period: int = self._pairlistconfig.get('refresh_period', 1800)	1✔
49	self._keep_pairlist_on_failure = self._pairlistconfig.get('keep_pairlist_on_failure', True)	1✔
50	self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)	1✔
51	self._pairlist_url = self._pairlistconfig.get('pairlist_url', '')	1✔
52	self._read_timeout = self._pairlistconfig.get('read_timeout', 60)	1✔
53	self._bearer_token = self._pairlistconfig.get('bearer_token', '')	1✔
54	self._init_done = False	1✔
55	self._save_to_file = self._pairlistconfig.get('save_to_file', None)	1✔
56	self._last_pairlist: List[Any] = list()	1✔
57
58	if self._mode not in ['whitelist', 'blacklist']:	1✔
59	raise OperationalException(	1✔
60	'`mode` not configured correctly. Supported Modes '
61	'are "whitelist","blacklist"')
62
63	if self._processing_mode not in ['filter', 'append']:	1✔
64	raise OperationalException(	1✔
65	'`processing_mode` not configured correctly. Supported Modes '
66	'are "filter","append"')
67
68	if self._pairlist_pos == 0 and self._mode == 'blacklist':	1✔
69	raise OperationalException(	1✔
70	'A `blacklist` mode RemotePairList can not be on the first '
71	'position of your pairlist.')
72
73	@property	1✔
74	def needstickers(self) -> bool:	1✔
75	"""
76	Boolean property defining if tickers are necessary.
77	If no Pairlist requires tickers, an empty Dict is passed
78	as tickers argument to filter_pairlist
79	"""
80	return False	1✔
81
82	def short_desc(self) -> str:	1✔
83	"""
84	Short whitelist method description - used for startup-messages
85	"""
86	return f"{self.name} - {self._pairlistconfig['number_assets']} pairs from RemotePairlist."	1✔
87
88	@staticmethod	1✔
89	def description() -> str:	1✔
90	return "Retrieve pairs from a remote API or local file."	1✔
91
92	@staticmethod	1✔
93	def available_parameters() -> Dict[str, PairlistParameter]:	1✔
94	return {	1✔
95	"pairlist_url": {
96	"type": "string",
97	"default": "",
98	"description": "URL to fetch pairlist from",
99	"help": "URL to fetch pairlist from",
100	},
101	"number_assets": {
102	"type": "number",
103	"default": 30,
104	"description": "Number of assets",
105	"help": "Number of assets to use from the pairlist.",
106	},
107	"mode": {
108	"type": "option",
109	"default": "whitelist",
110	"options": ["whitelist", "blacklist"],
111	"description": "Pairlist mode",
112	"help": "Should this pairlist operate as a whitelist or blacklist?",
113	},
114	"processing_mode": {
115	"type": "option",
116	"default": "filter",
117	"options": ["filter", "append"],
118	"description": "Processing mode",
119	"help": "Append pairs to incoming pairlist or filter them?",
120	},
121	**IPairList.refresh_period_parameter(),
122	"keep_pairlist_on_failure": {
123	"type": "boolean",
124	"default": True,
125	"description": "Keep last pairlist on failure",
126	"help": "Keep last pairlist on failure",
127	},
128	"read_timeout": {
129	"type": "number",
130	"default": 60,
131	"description": "Read timeout",
132	"help": "Request timeout for remote pairlist",
133	},
134	"bearer_token": {
135	"type": "string",
136	"default": "",
137	"description": "Bearer token",
138	"help": "Bearer token - used for auth against the upstream service.",
139	},
140	"save_to_file": {
141	"type": "string",
142	"default": "",
143	"description": "Filename to save processed pairlist to.",
144	"help": "Specify a filename to save the processed pairlist in JSON format.",
145	},
146	}
147
148	def process_json(self, jsonparse) -> List[str]:	1✔
149
150	pairlist = jsonparse.get('pairs', [])	1✔
151	remote_refresh_period = int(jsonparse.get('refresh_period', self._refresh_period))	1✔
152
153	if self._refresh_period < remote_refresh_period:	1✔
154	self.log_once(f'Refresh Period has been increased from {self._refresh_period}'	1✔
155	f' to minimum allowed: {remote_refresh_period} from Remote.', logger.info)
156
157	self._refresh_period = remote_refresh_period	1✔
158	self._pair_cache = TTLCache(maxsize=1, ttl=remote_refresh_period)	1✔
159
160	self._init_done = True	1✔
161
162	return pairlist	1✔
163
164	def return_last_pairlist(self) -> List[str]:	1✔
165	if self._keep_pairlist_on_failure:	1✔
166	pairlist = self._last_pairlist	1✔
167	self.log_once('Keeping last fetched pairlist', logger.info)	1✔
168	else:
169	pairlist = []	×
170
171	return pairlist	1✔
172
173	def fetch_pairlist(self) -> Tuple[List[str], float]:	1✔
174
175	headers = {	1✔
176	'User-Agent': 'Freqtrade/' + __version__ + ' Remotepairlist'
177	}
178
179	if self._bearer_token:	1✔
180	headers['Authorization'] = f'Bearer {self._bearer_token}'	×
181
182	try:	1✔
183	response = requests.get(self._pairlist_url, headers=headers,	1✔
184	timeout=self._read_timeout)
185	content_type = response.headers.get('content-type')	1✔
186	time_elapsed = response.elapsed.total_seconds()	1✔
187
188	if "application/json" in str(content_type):	1✔
189	jsonparse = response.json()	1✔
190
191	try:	1✔
192	pairlist = self.process_json(jsonparse)	1✔
193	except Exception as e:	×
194	pairlist = self._handle_error(f'Failed processing JSON data: {type(e)}')	×
195	else:
196	pairlist = self._handle_error(f'RemotePairList is not of type JSON.'	1✔
197	f' {self._pairlist_url}')
198
199	except requests.exceptions.RequestException:	1✔
200	pairlist = self._handle_error(f'Was not able to fetch pairlist from:'	1✔
201	f' {self._pairlist_url}')
202
203	time_elapsed = 0	1✔
204
205	return pairlist, time_elapsed	1✔
206
207	def _handle_error(self, error: str) -> List[str]:	1✔
208	if self._init_done:	1✔
209	self.log_once("Error: " + error, logger.info)	1✔
210	return self.return_last_pairlist()	1✔
211	else:
212	raise OperationalException(error)	1✔
213
214	def gen_pairlist(self, tickers: Tickers) -> List[str]:	1✔
215	"""
216	Generate the pairlist
217	:param tickers: Tickers (from exchange.get_tickers). May be cached.
218	:return: List of pairs
219	"""
220
221	if self._init_done:	1✔
222	pairlist = self._pair_cache.get('pairlist')	1✔
223	if pairlist == [None]:	1✔
224	# Valid but empty pairlist.
225	return []	×
226	else:
227	pairlist = []	1✔
228
229	time_elapsed = 0.0	1✔
230
231	if pairlist:	1✔
232	# Item found - no refresh necessary
233	return pairlist.copy()	×
234	else:
235	if self._pairlist_url.startswith("file:///"):	1✔
236	filename = self._pairlist_url.split("file:///", 1)[1]	1✔
237	file_path = Path(filename)	1✔
238
239	if file_path.exists():	1✔
240	with file_path.open() as json_file:	1✔
241	try:	1✔
242	# Load the JSON data into a dictionary
243	jsonparse = rapidjson.load(json_file, parse_mode=CONFIG_PARSE_MODE)	1✔
244	pairlist = self.process_json(jsonparse)	1✔
245	except Exception as e:	×
246	pairlist = self._handle_error(f'processing JSON data: {type(e)}')	×
247	else:
248	pairlist = self._handle_error(f"{self._pairlist_url} does not exist.")	×
249
250	else:
251	# Fetch Pairlist from Remote URL
252	pairlist, time_elapsed = self.fetch_pairlist()	1✔
253
254	self.log_once(f"Fetched pairs: {pairlist}", logger.debug)	1✔
255
256	pairlist = expand_pairlist(pairlist, list(self._exchange.get_markets().keys()))	1✔
257	pairlist = self._whitelist_for_active_markets(pairlist)	1✔
258	pairlist = pairlist[:self._number_pairs]	1✔
259
260	if pairlist:	1✔
261	self._pair_cache['pairlist'] = pairlist.copy()	1✔
262	else:
263	# If pairlist is empty, set a dummy value to avoid fetching again
264	self._pair_cache['pairlist'] = [None]	×
265
266	if time_elapsed != 0.0:	1✔
267	self.log_once(f'Pairlist Fetched in {time_elapsed} seconds.', logger.info)	1✔
268	else:
269	self.log_once('Fetched Pairlist.', logger.info)	1✔
270
271	self._last_pairlist = list(pairlist)	1✔
272
273	if self._save_to_file:	1✔
274	self.save_pairlist(pairlist, self._save_to_file)	×
275
276	return pairlist	1✔
277
278	def save_pairlist(self, pairlist: List[str], filename: str) -> None:	1✔
279	pairlist_data = {	×
280	"pairs": pairlist
281	}
282	try:	×
283	file_path = Path(filename)	×
284	with file_path.open('w') as json_file:	×
285	rapidjson.dump(pairlist_data, json_file)	×
286	logger.info(f"Processed pairlist saved to {filename}")	×
287	except Exception as e:	×
288	logger.error(f"Error saving processed pairlist to {filename}: {e}")	×
289
290	def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:	1✔
291	"""
292	Filters and sorts pairlist and returns the whitelist again.
293	Called on each bot iteration - please use internal caching if necessary
294	:param pairlist: pairlist to filter or sort
295	:param tickers: Tickers (from exchange.get_tickers). May be cached.
296	:return: new whitelist
297	"""
298	rpl_pairlist = self.gen_pairlist(tickers)	1✔
299	merged_list = []	1✔
300	filtered = []	1✔
301
302	if self._mode == "whitelist":	1✔
303	if self._processing_mode == "filter":	1✔
304	merged_list = [pair for pair in pairlist if pair in rpl_pairlist]	1✔
305	elif self._processing_mode == "append":	1✔
306	merged_list = pairlist + rpl_pairlist	1✔
307	merged_list = sorted(set(merged_list), key=merged_list.index)	1✔
308	else:
309	for pair in pairlist:	1✔
310	if pair not in rpl_pairlist:	1✔
311	merged_list.append(pair)	1✔
312	else:
313	filtered.append(pair)	1✔
314	if filtered:	1✔
315	self.log_once(f"Blacklist - Filtered out pairs: {filtered}", logger.info)	1✔
316
317	merged_list = merged_list[:self._number_pairs]	1✔
318	return merged_list	1✔

freqtrade / freqtrade / 9394559170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous