9394559170

Committed 26 Apr 2024 06:36AM UTC coverage: 94.656% (-0.02%) from 94.674%

Build # 9394559170

Build Type

push

github

Committed by

xmatthias

Commit Message

Loader should be passed as kwarg for clarity

Run Details

20280 of 21425 relevant lines covered (94.66%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

99.03

/freqtrade/plugins/pairlist/VolumePairList.py

"""
Volume PairList provider

Provides dynamic pair list based on trade volumes
"""
import logging
from datetime import timedelta
from typing import Any, Dict, List, Literal

from cachetools import TTLCache

from freqtrade.constants import Config, ListPairsWithTimeframes
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date
from freqtrade.exchange.types import Tickers
from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter
from freqtrade.util import dt_now, format_ms_time


logger = logging.getLogger(__name__)


SORT_VALUES = ['quoteVolume']


class VolumePairList(IPairList):

    is_pairlist_generator = True

    def __init__(self, exchange, pairlistmanager,
                 config: Config, pairlistconfig: Dict[str, Any],
                 pairlist_pos: int) -> None:
        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)

        if 'number_assets' not in self._pairlistconfig:
            raise OperationalException(
                '`number_assets` not specified. Please check your configuration '
                'for "pairlist.config.number_assets"')

        self._stake_currency = config['stake_currency']
        self._number_pairs = self._pairlistconfig['number_assets']
        self._sort_key: Literal['quoteVolume'] = self._pairlistconfig.get('sort_key', 'quoteVolume')
        self._min_value = self._pairlistconfig.get('min_value', 0)
        self._max_value = self._pairlistconfig.get("max_value", None)
        self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
        self._lookback_days = self._pairlistconfig.get('lookback_days', 0)
        self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')
        self._lookback_period = self._pairlistconfig.get('lookback_period', 0)
        self._def_candletype = self._config['candle_type_def']

        if (self._lookback_days > 0) & (self._lookback_period > 0):
            raise OperationalException(
                'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
                'config. Please set lookback_days only or lookback_period and lookback_timeframe '
                'and restart the bot.'
            )

        # overwrite lookback timeframe and days when lookback_days is set
        if self._lookback_days > 0:
            self._lookback_timeframe = '1d'
            self._lookback_period = self._lookback_days

        # get timeframe in minutes and seconds
        self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)
        _tf_in_sec = self._tf_in_min * 60

        # whether to use range lookback or not
        self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)

        if self._use_range & (self._refresh_period < _tf_in_sec):
            raise OperationalException(
                f'Refresh period of {self._refresh_period} seconds is smaller than one '
                f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
                f'to at least {_tf_in_sec} and restart the bot.'
            )

        if (not self._use_range and not (
                self._exchange.exchange_has('fetchTickers')
                and self._exchange.get_option("tickers_have_quoteVolume"))):
            raise OperationalException(
                "Exchange does not support dynamic whitelist in this configuration. "
                "Please edit your config and either remove Volumepairlist, "
                "or switch to using candles. and restart the bot."
            )

        if not self._validate_keys(self._sort_key):
            raise OperationalException(
                f'key {self._sort_key} not in {SORT_VALUES}')

        candle_limit = exchange.ohlcv_candle_limit(
            self._lookback_timeframe, self._config['candle_type_def'])
        if self._lookback_period < 0:
            raise OperationalException("VolumeFilter requires lookback_period to be >= 0")
        if self._lookback_period > candle_limit:
            raise OperationalException("VolumeFilter requires lookback_period to not "
                                       f"exceed exchange max request size ({candle_limit})")

    @property
    def needstickers(self) -> bool:
        """
        Boolean property defining if tickers are necessary.
        If no Pairlist requires tickers, an empty Dict is passed
        as tickers argument to filter_pairlist
        """
        return not self._use_range

    def _validate_keys(self, key):
        return key in SORT_VALUES

    def short_desc(self) -> str:
        """
        Short whitelist method description - used for startup-messages
        """
        return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."

    @staticmethod
    def description() -> str:
        return "Provides dynamic pair list based on trade volumes."

    @staticmethod
    def available_parameters() -> Dict[str, PairlistParameter]:
        return {
            "number_assets": {
                "type": "number",
                "default": 30,
                "description": "Number of assets",
                "help": "Number of assets to use from the pairlist",
            },
            "sort_key": {
                "type": "option",
                "default": "quoteVolume",
                "options": SORT_VALUES,
                "description": "Sort key",
                "help": "Sort key to use for sorting the pairlist.",
            },
            "min_value": {
                "type": "number",
                "default": 0,
                "description": "Minimum value",
                "help": "Minimum value to use for filtering the pairlist.",
            },
            "max_value": {
                "type": "number",
                "default": None,
                "description": "Maximum value",
                "help": "Maximum value to use for filtering the pairlist.",
            },
            **IPairList.refresh_period_parameter(),
            "lookback_days": {
                "type": "number",
                "default": 0,
                "description": "Lookback Days",
                "help": "Number of days to look back at.",
            },
            "lookback_timeframe": {
                "type": "string",
                "default": "",
                "description": "Lookback Timeframe",
                "help": "Timeframe to use for lookback.",
            },
            "lookback_period": {
                "type": "number",
                "default": 0,
                "description": "Lookback Period",
                "help": "Number of periods to look back at.",
            },
        }

    def gen_pairlist(self, tickers: Tickers) -> List[str]:
        """
        Generate the pairlist
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: List of pairs
        """
        # Generate dynamic whitelist
        # Must always run if this pairlist is not the first in the list.
        pairlist = self._pair_cache.get('pairlist')
        if pairlist:
            # Item found - no refresh necessary
            return pairlist.copy()
        else:
            # Use fresh pairlist
            # Check if pair quote currency equals to the stake currency.
            _pairlist = [k for k in self._exchange.get_markets(
                quote_currencies=[self._stake_currency],
                tradable_only=True, active_only=True).keys()]
            # No point in testing for blacklisted pairs...
            _pairlist = self.verify_blacklist(_pairlist, logger.info)
            if not self._use_range:
                filtered_tickers = [
                    v for k, v in tickers.items()
                    if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
                        and (self._use_range or v.get(self._sort_key) is not None)
                        and v['symbol'] in _pairlist)]
                pairlist = [s['symbol'] for s in filtered_tickers]
            else:
                pairlist = _pairlist

            pairlist = self.filter_pairlist(pairlist, tickers)
            self._pair_cache['pairlist'] = pairlist.copy()

        return pairlist

    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
        """
        Filters and sorts pairlist and returns the whitelist again.
        Called on each bot iteration - please use internal caching if necessary
        :param pairlist: pairlist to filter or sort
        :param tickers: Tickers (from exchange.get_tickers). May be cached.
        :return: new whitelist
        """
        if self._use_range:
            # Create bare minimum from tickers structure.
            filtered_tickers: List[Dict[str, Any]] = [{'symbol': k} for k in pairlist]

            # get lookback period in ms, for exchange ohlcv fetch
            since_ms = int(timeframe_to_prev_date(
                self._lookback_timeframe,
                dt_now() + timedelta(
                    minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
                    ).timestamp()) * 1000

            to_ms = int(timeframe_to_prev_date(
                            self._lookback_timeframe,
                            dt_now() - timedelta(minutes=self._tf_in_min)
                            ).timestamp()) * 1000

            # todo: utc date output for starting date
            self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "
                          f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
                          f"till {format_ms_time(to_ms)}", logger.info)
            needed_pairs: ListPairsWithTimeframes = [
                (p, self._lookback_timeframe, self._def_candletype) for p in
                [s['symbol'] for s in filtered_tickers]
                if p not in self._pair_cache
            ]

            candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)

            for i, p in enumerate(filtered_tickers):
                contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0
                pair_candles = candles[
                    (p['symbol'], self._lookback_timeframe, self._def_candletype)
                ] if (
                    p['symbol'], self._lookback_timeframe, self._def_candletype
                    ) in candles else None
                # in case of candle data calculate typical price and quoteVolume for candle
                if pair_candles is not None and not pair_candles.empty:
                    if self._exchange.get_option("ohlcv_volume_currency") == "base":
                        pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']
                                                         + pair_candles['close']) / 3

                        pair_candles['quoteVolume'] = (
                            pair_candles['volume'] * pair_candles['typical_price']
                            * contract_size
                        )
                    else:
                        # Exchange ohlcv data is in quote volume already.
                        pair_candles['quoteVolume'] = pair_candles['volume']
                    # ensure that a rolling sum over the lookback_period is built
                    # if pair_candles contains more candles than lookback_period
                    quoteVolume = (pair_candles['quoteVolume']
                                   .rolling(self._lookback_period)
                                   .sum()
                                   .fillna(0)
                                   .iloc[-1])

                    # replace quoteVolume with range quoteVolume sum calculated above
                    filtered_tickers[i]['quoteVolume'] = quoteVolume
                else:
                    filtered_tickers[i]['quoteVolume'] = 0
        else:
            # Tickers mode - filter based on incoming pairlist.
            filtered_tickers = [v for k, v in tickers.items() if k in pairlist]

        if self._min_value > 0:
            filtered_tickers = [
                v for v in filtered_tickers if v[self._sort_key] > self._min_value]
        if self._max_value is not None:
            filtered_tickers = [
                v for v in filtered_tickers if v[self._sort_key] < self._max_value]

        sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])

        # Validate whitelist to only have active market pairs
        pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])
        pairs = self.verify_blacklist(pairs, logmethod=logger.info)
        # Limit pairlist to the requested number of pairs
        pairs = pairs[:self._number_pairs]

        return pairs

1	"""
2	Volume PairList provider
3
4	Provides dynamic pair list based on trade volumes
5	"""
6	import logging	1✔
7	from datetime import timedelta	1✔
8	from typing import Any, Dict, List, Literal	1✔
9
10	from cachetools import TTLCache	1✔
11
12	from freqtrade.constants import Config, ListPairsWithTimeframes	1✔
13	from freqtrade.exceptions import OperationalException	1✔
14	from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date	1✔
15	from freqtrade.exchange.types import Tickers	1✔
16	from freqtrade.plugins.pairlist.IPairList import IPairList, PairlistParameter	1✔
17	from freqtrade.util import dt_now, format_ms_time	1✔
18
19
20	logger = logging.getLogger(__name__)	1✔
21
22
23	SORT_VALUES = ['quoteVolume']	1✔
24
25
26	class VolumePairList(IPairList):	1✔
27
28	is_pairlist_generator = True	1✔
29
30	def __init__(self, exchange, pairlistmanager,	1✔
31	config: Config, pairlistconfig: Dict[str, Any],
32	pairlist_pos: int) -> None:
33	super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)	1✔
34
35	if 'number_assets' not in self._pairlistconfig:	1✔
36	raise OperationalException(	1✔
37	'`number_assets` not specified. Please check your configuration '
38	'for "pairlist.config.number_assets"')
39
40	self._stake_currency = config['stake_currency']	1✔
41	self._number_pairs = self._pairlistconfig['number_assets']	1✔
42	self._sort_key: Literal['quoteVolume'] = self._pairlistconfig.get('sort_key', 'quoteVolume')	1✔
43	self._min_value = self._pairlistconfig.get('min_value', 0)	1✔
44	self._max_value = self._pairlistconfig.get("max_value", None)	1✔
45	self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)	1✔
46	self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)	1✔
47	self._lookback_days = self._pairlistconfig.get('lookback_days', 0)	1✔
48	self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d')	1✔
49	self._lookback_period = self._pairlistconfig.get('lookback_period', 0)	1✔
50	self._def_candletype = self._config['candle_type_def']	1✔
51
52	if (self._lookback_days > 0) & (self._lookback_period > 0):	1✔
53	raise OperationalException(	1✔
54	'Ambigous configuration: lookback_days and lookback_period both set in pairlist '
55	'config. Please set lookback_days only or lookback_period and lookback_timeframe '
56	'and restart the bot.'
57	)
58
59	# overwrite lookback timeframe and days when lookback_days is set
60	if self._lookback_days > 0:	1✔
61	self._lookback_timeframe = '1d'	1✔
62	self._lookback_period = self._lookback_days	1✔
63
64	# get timeframe in minutes and seconds
65	self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe)	1✔
66	_tf_in_sec = self._tf_in_min * 60	1✔
67
68	# whether to use range lookback or not
69	self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0)	1✔
70
71	if self._use_range & (self._refresh_period < _tf_in_sec):	1✔
72	raise OperationalException(	1✔
73	f'Refresh period of {self._refresh_period} seconds is smaller than one '
74	f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period '
75	f'to at least {_tf_in_sec} and restart the bot.'
76	)
77
78	if (not self._use_range and not (	1✔
79	self._exchange.exchange_has('fetchTickers')
80	and self._exchange.get_option("tickers_have_quoteVolume"))):
81	raise OperationalException(	1✔
82	"Exchange does not support dynamic whitelist in this configuration. "
83	"Please edit your config and either remove Volumepairlist, "
84	"or switch to using candles. and restart the bot."
85	)
86
87	if not self._validate_keys(self._sort_key):	1✔
88	raise OperationalException(	1✔
89	f'key {self._sort_key} not in {SORT_VALUES}')
90
91	candle_limit = exchange.ohlcv_candle_limit(	1✔
92	self._lookback_timeframe, self._config['candle_type_def'])
93	if self._lookback_period < 0:	1✔
94	raise OperationalException("VolumeFilter requires lookback_period to be >= 0")	1✔
95	if self._lookback_period > candle_limit:	1✔
96	raise OperationalException("VolumeFilter requires lookback_period to not "	1✔
97	f"exceed exchange max request size ({candle_limit})")
98
99	@property	1✔
100	def needstickers(self) -> bool:	1✔
101	"""
102	Boolean property defining if tickers are necessary.
103	If no Pairlist requires tickers, an empty Dict is passed
104	as tickers argument to filter_pairlist
105	"""
106	return not self._use_range	1✔
107
108	def _validate_keys(self, key):	1✔
109	return key in SORT_VALUES	1✔
110
111	def short_desc(self) -> str:	1✔
112	"""
113	Short whitelist method description - used for startup-messages
114	"""
115	return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."	1✔
116
117	@staticmethod	1✔
118	def description() -> str:	1✔
119	return "Provides dynamic pair list based on trade volumes."	1✔
120
121	@staticmethod	1✔
122	def available_parameters() -> Dict[str, PairlistParameter]:	1✔
123	return {	1✔
124	"number_assets": {
125	"type": "number",
126	"default": 30,
127	"description": "Number of assets",
128	"help": "Number of assets to use from the pairlist",
129	},
130	"sort_key": {
131	"type": "option",
132	"default": "quoteVolume",
133	"options": SORT_VALUES,
134	"description": "Sort key",
135	"help": "Sort key to use for sorting the pairlist.",
136	},
137	"min_value": {
138	"type": "number",
139	"default": 0,
140	"description": "Minimum value",
141	"help": "Minimum value to use for filtering the pairlist.",
142	},
143	"max_value": {
144	"type": "number",
145	"default": None,
146	"description": "Maximum value",
147	"help": "Maximum value to use for filtering the pairlist.",
148	},
149	**IPairList.refresh_period_parameter(),
150	"lookback_days": {
151	"type": "number",
152	"default": 0,
153	"description": "Lookback Days",
154	"help": "Number of days to look back at.",
155	},
156	"lookback_timeframe": {
157	"type": "string",
158	"default": "",
159	"description": "Lookback Timeframe",
160	"help": "Timeframe to use for lookback.",
161	},
162	"lookback_period": {
163	"type": "number",
164	"default": 0,
165	"description": "Lookback Period",
166	"help": "Number of periods to look back at.",
167	},
168	}
169
170	def gen_pairlist(self, tickers: Tickers) -> List[str]:	1✔
171	"""
172	Generate the pairlist
173	:param tickers: Tickers (from exchange.get_tickers). May be cached.
174	:return: List of pairs
175	"""
176	# Generate dynamic whitelist
177	# Must always run if this pairlist is not the first in the list.
178	pairlist = self._pair_cache.get('pairlist')	1✔
179	if pairlist:	1✔
180	# Item found - no refresh necessary
181	return pairlist.copy()	1✔
182	else:
183	# Use fresh pairlist
184	# Check if pair quote currency equals to the stake currency.
185	_pairlist = [k for k in self._exchange.get_markets(	1✔
186	quote_currencies=[self._stake_currency],
187	tradable_only=True, active_only=True).keys()]
188	# No point in testing for blacklisted pairs...
189	_pairlist = self.verify_blacklist(_pairlist, logger.info)	1✔
190	if not self._use_range:	1✔
191	filtered_tickers = [	1✔
192	v for k, v in tickers.items()
193	if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
194	and (self._use_range or v.get(self._sort_key) is not None)
195	and v['symbol'] in _pairlist)]
196	pairlist = [s['symbol'] for s in filtered_tickers]	1✔
197	else:
198	pairlist = _pairlist	1✔
199
200	pairlist = self.filter_pairlist(pairlist, tickers)	1✔
201	self._pair_cache['pairlist'] = pairlist.copy()	1✔
202
203	return pairlist	1✔
204
205	def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:	1✔
206	"""
207	Filters and sorts pairlist and returns the whitelist again.
208	Called on each bot iteration - please use internal caching if necessary
209	:param pairlist: pairlist to filter or sort
210	:param tickers: Tickers (from exchange.get_tickers). May be cached.
211	:return: new whitelist
212	"""
213	if self._use_range:	1✔
214	# Create bare minimum from tickers structure.
215	filtered_tickers: List[Dict[str, Any]] = [{'symbol': k} for k in pairlist]	1✔
216
217	# get lookback period in ms, for exchange ohlcv fetch
218	since_ms = int(timeframe_to_prev_date(	1✔
219	self._lookback_timeframe,
220	dt_now() + timedelta(
221	minutes=-(self._lookback_period * self._tf_in_min) - self._tf_in_min)
222	).timestamp()) * 1000
223
224	to_ms = int(timeframe_to_prev_date(	1✔
225	self._lookback_timeframe,
226	dt_now() - timedelta(minutes=self._tf_in_min)
227	).timestamp()) * 1000
228
229	# todo: utc date output for starting date
230	self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: "	1✔
231	f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} "
232	f"till {format_ms_time(to_ms)}", logger.info)
233	needed_pairs: ListPairsWithTimeframes = [	1✔
234	(p, self._lookback_timeframe, self._def_candletype) for p in
235	[s['symbol'] for s in filtered_tickers]
236	if p not in self._pair_cache
237	]
238
239	candles = self._exchange.refresh_ohlcv_with_cache(needed_pairs, since_ms)	1✔
240
241	for i, p in enumerate(filtered_tickers):	1✔
242	contract_size = self._exchange.markets[p['symbol']].get('contractSize', 1.0) or 1.0	1✔
243	pair_candles = candles[	1✔
244	(p['symbol'], self._lookback_timeframe, self._def_candletype)
245	] if (
246	p['symbol'], self._lookback_timeframe, self._def_candletype
247	) in candles else None
248	# in case of candle data calculate typical price and quoteVolume for candle
249	if pair_candles is not None and not pair_candles.empty:	1✔
250	if self._exchange.get_option("ohlcv_volume_currency") == "base":	1✔
251	pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low']	1✔
252	+ pair_candles['close']) / 3
253
254	pair_candles['quoteVolume'] = (	1✔
255	pair_candles['volume'] * pair_candles['typical_price']
256	* contract_size
257	)
258	else:
259	# Exchange ohlcv data is in quote volume already.
260	pair_candles['quoteVolume'] = pair_candles['volume']	×
261	# ensure that a rolling sum over the lookback_period is built
262	# if pair_candles contains more candles than lookback_period
263	quoteVolume = (pair_candles['quoteVolume']	1✔
264	.rolling(self._lookback_period)
265	.sum()
266	.fillna(0)
267	.iloc[-1])
268
269	# replace quoteVolume with range quoteVolume sum calculated above
270	filtered_tickers[i]['quoteVolume'] = quoteVolume	1✔
271	else:
272	filtered_tickers[i]['quoteVolume'] = 0	1✔
273	else:
274	# Tickers mode - filter based on incoming pairlist.
275	filtered_tickers = [v for k, v in tickers.items() if k in pairlist]	1✔
276
277	if self._min_value > 0:	1✔
278	filtered_tickers = [	1✔
279	v for v in filtered_tickers if v[self._sort_key] > self._min_value]
280	if self._max_value is not None:	1✔
281	filtered_tickers = [	1✔
282	v for v in filtered_tickers if v[self._sort_key] < self._max_value]
283
284	sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key])	1✔
285
286	# Validate whitelist to only have active market pairs
287	pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers])	1✔
288	pairs = self.verify_blacklist(pairs, logmethod=logger.info)	1✔
289	# Limit pairlist to the requested number of pairs
290	pairs = pairs[:self._number_pairs]	1✔
291
292	return pairs	1✔

freqtrade / freqtrade / 9394559170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous