4131167254

Build Type

push

github-actions

Committed by GitHub

Commit Message

Merge pull request #7983 from stash86/bt-metrics

Run Details

16866 of 17748 relevant lines covered (95.03%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.67

/freqtrade/freqai/utils.py

import logging
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict

import numpy as np
import pandas as pd
import rapidjson

from freqtrade.configuration import TimeRange
from freqtrade.constants import Config
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.exchange.exchange import market_is_active
from freqtrade.freqai.data_drawer import FreqaiDataDrawer
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist


logger = logging.getLogger(__name__)


def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
    """
    Called only once upon start of bot to download the necessary data for
    populating indicators and training the model.
    :param timerange: TimeRange = The full data timerange for populating the indicators
                                    and training the model.
    :param dp: DataProvider instance attached to the strategy
    """

    if dp._exchange is None:
        raise OperationalException('No exchange object found.')
    markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
               or config.get('include_inactive')]

    all_pairs = dynamic_expand_pairlist(config, markets)

    timerange = get_required_data_timerange(config)

    new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)

    refresh_backtest_ohlcv_data(
        dp._exchange,
        pairs=all_pairs,
        timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
        datadir=config["datadir"],
        timerange=timerange,
        new_pairs_days=new_pairs_days,
        erase=False,
        data_format=config.get("dataformat_ohlcv", "json"),
        trading_mode=config.get("trading_mode", "spot"),
        prepend=config.get("prepend_data", False),
    )


def get_required_data_timerange(config: Config) -> TimeRange:
    """
    Used to compute the required data download time range
    for auto data-download in FreqAI
    """
    time = datetime.now(tz=timezone.utc).timestamp()

    timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")

    max_tf_seconds = 0
    for tf in timeframes:
        secs = timeframe_to_seconds(tf)
        if secs > max_tf_seconds:
            max_tf_seconds = secs

    startup_candles = config.get('startup_candle_count', 0)
    indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"]

    # factor the max_period as a factor of safety.
    max_period = int(max(startup_candles, max(indicator_periods)) * 1.5)
    config['startup_candle_count'] = max_period
    logger.info(f'FreqAI auto-downloader using {max_period} startup candles.')

    additional_seconds = max_period * max_tf_seconds

    startts = int(
        time
        - config["freqai"].get("train_period_days", 0) * 86400
        - additional_seconds
    )
    stopts = int(time)
    data_load_timerange = TimeRange('date', 'date', startts, stopts)

    return data_load_timerange


# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
# def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
#     """
#     Called only once upon start of bot to download the necessary data for
#     populating indicators and training a FreqAI model.
#     :param timerange: TimeRange = The full data timerange for populating the indicators
#                                     and training the model.
#     :param dp: DataProvider instance attached to the strategy
#     """

#     if dp._exchange is not None:
#         markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
#                    or config.get('include_inactive')]
#     else:
#         # This should not occur:
#         raise OperationalException('No exchange object found.')

#     all_pairs = dynamic_expand_pairlist(config, markets)

#     if not dp._exchange:
#         # Not realistic - this is only called in live mode.
#         raise OperationalException("Dataprovider did not have an exchange attached.")

#     time = datetime.now(tz=timezone.utc).timestamp()

#     for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
#         timerange = TimeRange()
#         timerange.startts = int(time)
#         timerange.stopts = int(time)
#         startup_candles = dp.get_required_startup(str(tf))
#         tf_seconds = timeframe_to_seconds(str(tf))
#         timerange.subtract_start(tf_seconds * startup_candles)
#         new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)
#         # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
#         # redownloads the funding rate for each pair.
#         refresh_backtest_ohlcv_data(
#             dp._exchange,
#             pairs=all_pairs,
#             timeframes=[tf],
#             datadir=config["datadir"],
#             timerange=timerange,
#             new_pairs_days=new_pairs_days,
#             erase=False,
#             data_format=config.get("dataformat_ohlcv", "json"),
#             trading_mode=config.get("trading_mode", "spot"),
#             prepend=config.get("prepend_data", False),
#         )


def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
                            count_max: int = 25) -> None:
    """
        Plot Best and worst features by importance for a single sub-train.
        :param model: Any = A model which was `fit` using a common library
                            such as catboost or lightgbm
        :param pair: str = pair e.g. BTC/USD
        :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
        :param count_max: int = the amount of features to be loaded per column
    """
    from freqtrade.plot.plotting import go, make_subplots, store_plot_file

    # Extract feature importance from model
    models = {}
    if 'FreqaiMultiOutputRegressor' in str(model.__class__):
        for estimator, label in zip(model.estimators_, dk.label_list):
            models[label] = estimator
    else:
        models[dk.label_list[0]] = model

    for label in models:
        mdl = models[label]
        if "catboost.core" in str(mdl.__class__):
            feature_importance = mdl.get_feature_importance()
        elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__):
            feature_importance = mdl.feature_importances_
        else:
            logger.info('Model type not support for generating feature importances.')
            return

        # Data preparation
        fi_df = pd.DataFrame({
            "feature_names": np.array(dk.data_dictionary['train_features'].columns),
            "feature_importance": np.array(feature_importance)
        })
        fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1]
        fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1]

        # Plotting
        def add_feature_trace(fig, fi_df, col):
            return fig.add_trace(
                go.Bar(
                    x=fi_df["feature_importance"],
                    y=fi_df["feature_names"],
                    orientation='h', showlegend=False
                ), row=1, col=col
            )
        fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5)
        fig = add_feature_trace(fig, fi_df_top, 1)
        fig = add_feature_trace(fig, fi_df_worst, 2)
        fig.update_layout(title_text=f"Best and worst features by importance {pair}")
        label = label.replace('&', '').replace('%', '')  # escape two FreqAI specific characters
        store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path)


def record_params(config: Dict[str, Any], full_path: Path) -> None:
    """
    Records run params in the full path for reproducibility
    """
    params_record_path = full_path / "run_params.json"

    run_params = {
        "freqai": config.get('freqai', {}),
        "timeframe": config.get('timeframe'),
        "stake_amount": config.get('stake_amount'),
        "stake_currency": config.get('stake_currency'),
        "max_open_trades": config.get('max_open_trades'),
        "pairs": config.get('exchange', {}).get('pair_whitelist')
    }

    with open(params_record_path, "w") as handle:
        rapidjson.dump(
            run_params,
            handle,
            indent=4,
            default=str,
            number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
        )


def get_timerange_backtest_live_models(config: Config) -> str:
    """
    Returns a formated timerange for backtest live/ready models
    :param config: Configuration dictionary

    :return: a string timerange (format example: '20220801-20220822')
    """
    dk = FreqaiDataKitchen(config)
    models_path = dk.get_full_models_path(config)
    dd = FreqaiDataDrawer(models_path, config)
    timerange = dd.get_timerange_from_live_historic_predictions()
    return timerange.timerange_str

1	import logging	1✔
2	from datetime import datetime, timezone	1✔
3	from pathlib import Path	1✔
4	from typing import Any, Dict	1✔
5
6	import numpy as np	1✔
7	import pandas as pd	1✔
8	import rapidjson	1✔
9
10	from freqtrade.configuration import TimeRange	1✔
11	from freqtrade.constants import Config	1✔
12	from freqtrade.data.dataprovider import DataProvider	1✔
13	from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data	1✔
14	from freqtrade.exceptions import OperationalException	1✔
15	from freqtrade.exchange import timeframe_to_seconds	1✔
16	from freqtrade.exchange.exchange import market_is_active	1✔
17	from freqtrade.freqai.data_drawer import FreqaiDataDrawer	1✔
18	from freqtrade.freqai.data_kitchen import FreqaiDataKitchen	1✔
19	from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist	1✔
20
21
22	logger = logging.getLogger(__name__)	1✔
23
24
25	def download_all_data_for_training(dp: DataProvider, config: Config) -> None:	1✔
26	"""
27	Called only once upon start of bot to download the necessary data for
28	populating indicators and training the model.
29	:param timerange: TimeRange = The full data timerange for populating the indicators
30	and training the model.
31	:param dp: DataProvider instance attached to the strategy
32	"""
33
34	if dp._exchange is None:	1✔
35	raise OperationalException('No exchange object found.')	×
36	markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)	1✔
37	or config.get('include_inactive')]
38
39	all_pairs = dynamic_expand_pairlist(config, markets)	1✔
40
41	timerange = get_required_data_timerange(config)	1✔
42
43	new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)	1✔
44
45	refresh_backtest_ohlcv_data(	1✔
46	dp._exchange,
47	pairs=all_pairs,
48	timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
49	datadir=config["datadir"],
50	timerange=timerange,
51	new_pairs_days=new_pairs_days,
52	erase=False,
53	data_format=config.get("dataformat_ohlcv", "json"),
54	trading_mode=config.get("trading_mode", "spot"),
55	prepend=config.get("prepend_data", False),
56	)
57
58
59	def get_required_data_timerange(config: Config) -> TimeRange:	1✔
60	"""
61	Used to compute the required data download time range
62	for auto data-download in FreqAI
63	"""
64	time = datetime.now(tz=timezone.utc).timestamp()	1✔
65
66	timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")	1✔
67
68	max_tf_seconds = 0	1✔
69	for tf in timeframes:	1✔
70	secs = timeframe_to_seconds(tf)	1✔
71	if secs > max_tf_seconds:	1✔
72	max_tf_seconds = secs	1✔
73
74	startup_candles = config.get('startup_candle_count', 0)	1✔
75	indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"]	1✔
76
77	# factor the max_period as a factor of safety.
78	max_period = int(max(startup_candles, max(indicator_periods)) * 1.5)	1✔
79	config['startup_candle_count'] = max_period	1✔
80	logger.info(f'FreqAI auto-downloader using {max_period} startup candles.')	1✔
81
82	additional_seconds = max_period * max_tf_seconds	1✔
83
84	startts = int(	1✔
85	time
86	- config["freqai"].get("train_period_days", 0) * 86400
87	- additional_seconds
88	)
89	stopts = int(time)	1✔
90	data_load_timerange = TimeRange('date', 'date', startts, stopts)	1✔
91
92	return data_load_timerange	1✔
93
94
95	# Keep below for when we wish to download heterogeneously lengthed data for FreqAI.
96	# def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
97	# """
98	# Called only once upon start of bot to download the necessary data for
99	# populating indicators and training a FreqAI model.
100	# :param timerange: TimeRange = The full data timerange for populating the indicators
101	# and training the model.
102	# :param dp: DataProvider instance attached to the strategy
103	# """
104
105	# if dp._exchange is not None:
106	# markets = [p for p, m in dp._exchange.markets.items() if market_is_active(m)
107	# or config.get('include_inactive')]
108	# else:
109	# # This should not occur:
110	# raise OperationalException('No exchange object found.')
111
112	# all_pairs = dynamic_expand_pairlist(config, markets)
113
114	# if not dp._exchange:
115	# # Not realistic - this is only called in live mode.
116	# raise OperationalException("Dataprovider did not have an exchange attached.")
117
118	# time = datetime.now(tz=timezone.utc).timestamp()
119
120	# for tf in config["freqai"]["feature_parameters"].get("include_timeframes"):
121	# timerange = TimeRange()
122	# timerange.startts = int(time)
123	# timerange.stopts = int(time)
124	# startup_candles = dp.get_required_startup(str(tf))
125	# tf_seconds = timeframe_to_seconds(str(tf))
126	# timerange.subtract_start(tf_seconds * startup_candles)
127	# new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)
128	# # FIXME: now that we are looping on `refresh_backtest_ohlcv_data`, the function
129	# # redownloads the funding rate for each pair.
130	# refresh_backtest_ohlcv_data(
131	# dp._exchange,
132	# pairs=all_pairs,
133	# timeframes=[tf],
134	# datadir=config["datadir"],
135	# timerange=timerange,
136	# new_pairs_days=new_pairs_days,
137	# erase=False,
138	# data_format=config.get("dataformat_ohlcv", "json"),
139	# trading_mode=config.get("trading_mode", "spot"),
140	# prepend=config.get("prepend_data", False),
141	# )
142
143
144	def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,	1✔
145	count_max: int = 25) -> None:
146	"""
147	Plot Best and worst features by importance for a single sub-train.
148	:param model: Any = A model which was `fit` using a common library
149	such as catboost or lightgbm
150	:param pair: str = pair e.g. BTC/USD
151	:param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
152	:param count_max: int = the amount of features to be loaded per column
153	"""
154	from freqtrade.plot.plotting import go, make_subplots, store_plot_file	1✔
155
156	# Extract feature importance from model
157	models = {}	1✔
158	if 'FreqaiMultiOutputRegressor' in str(model.__class__):	1✔
159	for estimator, label in zip(model.estimators_, dk.label_list):	×
160	models[label] = estimator	×
161	else:
162	models[dk.label_list[0]] = model	1✔
163
164	for label in models:	1✔
165	mdl = models[label]	1✔
166	if "catboost.core" in str(mdl.__class__):	1✔
167	feature_importance = mdl.get_feature_importance()	×
168	elif "lightgbm.sklearn" or "xgb" in str(mdl.__class__):	1✔
169	feature_importance = mdl.feature_importances_	1✔
170	else:
171	logger.info('Model type not support for generating feature importances.')
172	return
173
174	# Data preparation
175	fi_df = pd.DataFrame({	1✔
176	"feature_names": np.array(dk.data_dictionary['train_features'].columns),
177	"feature_importance": np.array(feature_importance)
178	})
179	fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1]	1✔
180	fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1]	1✔
181
182	# Plotting
183	def add_feature_trace(fig, fi_df, col):	1✔
184	return fig.add_trace(	1✔
185	go.Bar(
186	x=fi_df["feature_importance"],
187	y=fi_df["feature_names"],
188	orientation='h', showlegend=False
189	), row=1, col=col
190	)
191	fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5)	1✔
192	fig = add_feature_trace(fig, fi_df_top, 1)	1✔
193	fig = add_feature_trace(fig, fi_df_worst, 2)	1✔
194	fig.update_layout(title_text=f"Best and worst features by importance {pair}")	1✔
195	label = label.replace('&', '').replace('%', '') # escape two FreqAI specific characters	1✔
196	store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path)	1✔
197
198
199	def record_params(config: Dict[str, Any], full_path: Path) -> None:	1✔
200	"""
201	Records run params in the full path for reproducibility
202	"""
203	params_record_path = full_path / "run_params.json"	1✔
204
205	run_params = {	1✔
206	"freqai": config.get('freqai', {}),
207	"timeframe": config.get('timeframe'),
208	"stake_amount": config.get('stake_amount'),
209	"stake_currency": config.get('stake_currency'),
210	"max_open_trades": config.get('max_open_trades'),
211	"pairs": config.get('exchange', {}).get('pair_whitelist')
212	}
213
214	with open(params_record_path, "w") as handle:	1✔
215	rapidjson.dump(	1✔
216	run_params,
217	handle,
218	indent=4,
219	default=str,
220	number_mode=rapidjson.NM_NATIVE \| rapidjson.NM_NAN
221	)
222
223
224	def get_timerange_backtest_live_models(config: Config) -> str:	1✔
225	"""
226	Returns a formated timerange for backtest live/ready models
227	:param config: Configuration dictionary
228
229	:return: a string timerange (format example: '20220801-20220822')
230	"""
231	dk = FreqaiDataKitchen(config)	1✔
232	models_path = dk.get_full_models_path(config)	1✔
233	dd = FreqaiDataDrawer(models_path, config)	1✔
234	timerange = dd.get_timerange_from_live_historic_predictions()	1✔
235	return timerange.timerange_str	×

freqtrade / freqtrade / 4131167254

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous