9394559170

Committed 26 Apr 2024 06:36AM UTC coverage: 94.656% (-0.02%) from 94.674%

Build # 9394559170

Build Type

push

github

Committed by

xmatthias

Commit Message

Loader should be passed as kwarg for clarity

Run Details

20280 of 21425 relevant lines covered (94.66%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.77

/freqtrade/freqai/utils.py

import logging
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict

import numpy as np
import pandas as pd
import rapidjson

from freqtrade.configuration import TimeRange
from freqtrade.constants import Config
from freqtrade.data.dataprovider import DataProvider
from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_seconds
from freqtrade.freqai.data_drawer import FreqaiDataDrawer
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist


logger = logging.getLogger(__name__)


def download_all_data_for_training(dp: DataProvider, config: Config) -> None:
    """
    Called only once upon start of bot to download the necessary data for
    populating indicators and training the model.
    :param timerange: TimeRange = The full data timerange for populating the indicators
                                    and training the model.
    :param dp: DataProvider instance attached to the strategy
    """

    if dp._exchange is None:
        raise OperationalException('No exchange object found.')
    markets = [
        p for p in dp._exchange.get_markets(
            tradable_only=True, active_only=not config.get('include_inactive')
            ).keys()
    ]

    all_pairs = dynamic_expand_pairlist(config, markets)

    timerange = get_required_data_timerange(config)

    new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)

    refresh_backtest_ohlcv_data(
        dp._exchange,
        pairs=all_pairs,
        timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
        datadir=config["datadir"],
        timerange=timerange,
        new_pairs_days=new_pairs_days,
        erase=False,
        data_format=config.get("dataformat_ohlcv", "feather"),
        trading_mode=config.get("trading_mode", "spot"),
        prepend=config.get("prepend_data", False),
    )


def get_required_data_timerange(config: Config) -> TimeRange:
    """
    Used to compute the required data download time range
    for auto data-download in FreqAI
    """
    time = datetime.now(tz=timezone.utc).timestamp()

    timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")

    max_tf_seconds = 0
    for tf in timeframes:
        secs = timeframe_to_seconds(tf)
        if secs > max_tf_seconds:
            max_tf_seconds = secs

    startup_candles = config.get('startup_candle_count', 0)
    indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"]

    # factor the max_period as a factor of safety.
    max_period = int(max(startup_candles, max(indicator_periods)) * 1.5)
    config['startup_candle_count'] = max_period
    logger.info(f'FreqAI auto-downloader using {max_period} startup candles.')

    additional_seconds = max_period * max_tf_seconds

    startts = int(
        time
        - config["freqai"].get("train_period_days", 0) * 86400
        - additional_seconds
    )
    stopts = int(time)
    data_load_timerange = TimeRange('date', 'date', startts, stopts)

    return data_load_timerange


def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,
                            count_max: int = 25) -> None:
    """
        Plot Best and worst features by importance for a single sub-train.
        :param model: Any = A model which was `fit` using a common library
                            such as catboost or lightgbm
        :param pair: str = pair e.g. BTC/USD
        :param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
        :param count_max: int = the amount of features to be loaded per column
    """
    from freqtrade.plot.plotting import go, make_subplots, store_plot_file

    # Extract feature importance from model
    models = {}
    if 'FreqaiMultiOutputRegressor' in str(model.__class__):
        for estimator, label in zip(model.estimators_, dk.label_list):
            models[label] = estimator
    else:
        models[dk.label_list[0]] = model

    for label in models:
        mdl = models[label]
        if "catboost.core" in str(mdl.__class__):
            feature_importance = mdl.get_feature_importance()
        elif "lightgbm.sklearn" in str(mdl.__class__):
            feature_importance = mdl.feature_importances_
        elif "xgb" in str(mdl.__class__):
            feature_importance = mdl.feature_importances_
        else:
            logger.info('Model type does not support generating feature importances.')
            return

        # Data preparation
        fi_df = pd.DataFrame({
            "feature_names": np.array(dk.data_dictionary['train_features'].columns),
            "feature_importance": np.array(feature_importance)
        })
        fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1]
        fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1]

        # Plotting
        def add_feature_trace(fig, fi_df, col):
            return fig.add_trace(
                go.Bar(
                    x=fi_df["feature_importance"],
                    y=fi_df["feature_names"],
                    orientation='h', showlegend=False
                ), row=1, col=col
            )
        fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5)
        fig = add_feature_trace(fig, fi_df_top, 1)
        fig = add_feature_trace(fig, fi_df_worst, 2)
        fig.update_layout(title_text=f"Best and worst features by importance {pair}")
        label = label.replace('&', '').replace('%', '')  # escape two FreqAI specific characters
        store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path)


def record_params(config: Dict[str, Any], full_path: Path) -> None:
    """
    Records run params in the full path for reproducibility
    """
    params_record_path = full_path / "run_params.json"

    run_params = {
        "freqai": config.get('freqai', {}),
        "timeframe": config.get('timeframe'),
        "stake_amount": config.get('stake_amount'),
        "stake_currency": config.get('stake_currency'),
        "max_open_trades": config.get('max_open_trades'),
        "pairs": config.get('exchange', {}).get('pair_whitelist')
    }

    with params_record_path.open("w") as handle:
        rapidjson.dump(
            run_params,
            handle,
            indent=4,
            default=str,
            number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
        )


def get_timerange_backtest_live_models(config: Config) -> str:
    """
    Returns a formatted timerange for backtest live/ready models
    :param config: Configuration dictionary

    :return: a string timerange (format example: '20220801-20220822')
    """
    dk = FreqaiDataKitchen(config)
    models_path = dk.get_full_models_path(config)
    dd = FreqaiDataDrawer(models_path, config)
    timerange = dd.get_timerange_from_live_historic_predictions()
    return timerange.timerange_str


def get_tb_logger(model_type: str, path: Path, activate: bool) -> Any:

    if model_type == "pytorch" and activate:
        from freqtrade.freqai.tensorboard import TBLogger
        return TBLogger(path, activate)
    else:
        from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger
        return BaseTensorboardLogger(path, activate)

1	import logging	1✔
2	from datetime import datetime, timezone	1✔
3	from pathlib import Path	1✔
4	from typing import Any, Dict	1✔
5
6	import numpy as np	1✔
7	import pandas as pd	1✔
8	import rapidjson	1✔
9
10	from freqtrade.configuration import TimeRange	1✔
11	from freqtrade.constants import Config	1✔
12	from freqtrade.data.dataprovider import DataProvider	1✔
13	from freqtrade.data.history.history_utils import refresh_backtest_ohlcv_data	1✔
14	from freqtrade.exceptions import OperationalException	1✔
15	from freqtrade.exchange import timeframe_to_seconds	1✔
16	from freqtrade.freqai.data_drawer import FreqaiDataDrawer	1✔
17	from freqtrade.freqai.data_kitchen import FreqaiDataKitchen	1✔
18	from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist	1✔
19
20
21	logger = logging.getLogger(__name__)	1✔
22
23
24	def download_all_data_for_training(dp: DataProvider, config: Config) -> None:	1✔
25	"""
26	Called only once upon start of bot to download the necessary data for
27	populating indicators and training the model.
28	:param timerange: TimeRange = The full data timerange for populating the indicators
29	and training the model.
30	:param dp: DataProvider instance attached to the strategy
31	"""
32
33	if dp._exchange is None:	1✔
34	raise OperationalException('No exchange object found.')	×
35	markets = [	1✔
36	p for p in dp._exchange.get_markets(
37	tradable_only=True, active_only=not config.get('include_inactive')
38	).keys()
39	]
40
41	all_pairs = dynamic_expand_pairlist(config, markets)	1✔
42
43	timerange = get_required_data_timerange(config)	1✔
44
45	new_pairs_days = int((timerange.stopts - timerange.startts) / 86400)	1✔
46
47	refresh_backtest_ohlcv_data(	1✔
48	dp._exchange,
49	pairs=all_pairs,
50	timeframes=config["freqai"]["feature_parameters"].get("include_timeframes"),
51	datadir=config["datadir"],
52	timerange=timerange,
53	new_pairs_days=new_pairs_days,
54	erase=False,
55	data_format=config.get("dataformat_ohlcv", "feather"),
56	trading_mode=config.get("trading_mode", "spot"),
57	prepend=config.get("prepend_data", False),
58	)
59
60
61	def get_required_data_timerange(config: Config) -> TimeRange:	1✔
62	"""
63	Used to compute the required data download time range
64	for auto data-download in FreqAI
65	"""
66	time = datetime.now(tz=timezone.utc).timestamp()	1✔
67
68	timeframes = config["freqai"]["feature_parameters"].get("include_timeframes")	1✔
69
70	max_tf_seconds = 0	1✔
71	for tf in timeframes:	1✔
72	secs = timeframe_to_seconds(tf)	1✔
73	if secs > max_tf_seconds:	1✔
74	max_tf_seconds = secs	1✔
75
76	startup_candles = config.get('startup_candle_count', 0)	1✔
77	indicator_periods = config["freqai"]["feature_parameters"]["indicator_periods_candles"]	1✔
78
79	# factor the max_period as a factor of safety.
80	max_period = int(max(startup_candles, max(indicator_periods)) * 1.5)	1✔
81	config['startup_candle_count'] = max_period	1✔
82	logger.info(f'FreqAI auto-downloader using {max_period} startup candles.')	1✔
83
84	additional_seconds = max_period * max_tf_seconds	1✔
85
86	startts = int(	1✔
87	time
88	- config["freqai"].get("train_period_days", 0) * 86400
89	- additional_seconds
90	)
91	stopts = int(time)	1✔
92	data_load_timerange = TimeRange('date', 'date', startts, stopts)	1✔
93
94	return data_load_timerange	1✔
95
96
97	def plot_feature_importance(model: Any, pair: str, dk: FreqaiDataKitchen,	1✔
98	count_max: int = 25) -> None:
99	"""
100	Plot Best and worst features by importance for a single sub-train.
101	:param model: Any = A model which was `fit` using a common library
102	such as catboost or lightgbm
103	:param pair: str = pair e.g. BTC/USD
104	:param dk: FreqaiDataKitchen = non-persistent data container for current coin/loop
105	:param count_max: int = the amount of features to be loaded per column
106	"""
107	from freqtrade.plot.plotting import go, make_subplots, store_plot_file	1✔
108
109	# Extract feature importance from model
110	models = {}	1✔
111	if 'FreqaiMultiOutputRegressor' in str(model.__class__):	1✔
112	for estimator, label in zip(model.estimators_, dk.label_list):	×
113	models[label] = estimator	×
114	else:
115	models[dk.label_list[0]] = model	1✔
116
117	for label in models:	1✔
118	mdl = models[label]	1✔
119	if "catboost.core" in str(mdl.__class__):	1✔
120	feature_importance = mdl.get_feature_importance()	×
121	elif "lightgbm.sklearn" in str(mdl.__class__):	1✔
122	feature_importance = mdl.feature_importances_	1✔
123	elif "xgb" in str(mdl.__class__):	×
124	feature_importance = mdl.feature_importances_	×
125	else:
126	logger.info('Model type does not support generating feature importances.')	×
127	return	×
128
129	# Data preparation
130	fi_df = pd.DataFrame({	1✔
131	"feature_names": np.array(dk.data_dictionary['train_features'].columns),
132	"feature_importance": np.array(feature_importance)
133	})
134	fi_df_top = fi_df.nlargest(count_max, "feature_importance")[::-1]	1✔
135	fi_df_worst = fi_df.nsmallest(count_max, "feature_importance")[::-1]	1✔
136
137	# Plotting
138	def add_feature_trace(fig, fi_df, col):	1✔
139	return fig.add_trace(	1✔
140	go.Bar(
141	x=fi_df["feature_importance"],
142	y=fi_df["feature_names"],
143	orientation='h', showlegend=False
144	), row=1, col=col
145	)
146	fig = make_subplots(rows=1, cols=2, horizontal_spacing=0.5)	1✔
147	fig = add_feature_trace(fig, fi_df_top, 1)	1✔
148	fig = add_feature_trace(fig, fi_df_worst, 2)	1✔
149	fig.update_layout(title_text=f"Best and worst features by importance {pair}")	1✔
150	label = label.replace('&', '').replace('%', '') # escape two FreqAI specific characters	1✔
151	store_plot_file(fig, f"{dk.model_filename}-{label}.html", dk.data_path)	1✔
152
153
154	def record_params(config: Dict[str, Any], full_path: Path) -> None:	1✔
155	"""
156	Records run params in the full path for reproducibility
157	"""
158	params_record_path = full_path / "run_params.json"	1✔
159
160	run_params = {	1✔
161	"freqai": config.get('freqai', {}),
162	"timeframe": config.get('timeframe'),
163	"stake_amount": config.get('stake_amount'),
164	"stake_currency": config.get('stake_currency'),
165	"max_open_trades": config.get('max_open_trades'),
166	"pairs": config.get('exchange', {}).get('pair_whitelist')
167	}
168
169	with params_record_path.open("w") as handle:	1✔
170	rapidjson.dump(	1✔
171	run_params,
172	handle,
173	indent=4,
174	default=str,
175	number_mode=rapidjson.NM_NATIVE \| rapidjson.NM_NAN
176	)
177
178
179	def get_timerange_backtest_live_models(config: Config) -> str:	1✔
180	"""
181	Returns a formatted timerange for backtest live/ready models
182	:param config: Configuration dictionary
183
184	:return: a string timerange (format example: '20220801-20220822')
185	"""
186	dk = FreqaiDataKitchen(config)	1✔
187	models_path = dk.get_full_models_path(config)	1✔
188	dd = FreqaiDataDrawer(models_path, config)	1✔
189	timerange = dd.get_timerange_from_live_historic_predictions()	1✔
190	return timerange.timerange_str	×
191
192
193	def get_tb_logger(model_type: str, path: Path, activate: bool) -> Any:	1✔
194
195	if model_type == "pytorch" and activate:	1✔
196	from freqtrade.freqai.tensorboard import TBLogger	1✔
197	return TBLogger(path, activate)	1✔
198	else:
199	from freqtrade.freqai.tensorboard.base_tensorboard import BaseTensorboardLogger	1✔
200	return BaseTensorboardLogger(path, activate)	1✔

freqtrade / freqtrade / 9394559170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous