4131164979

Build Type

push

github-actions

Committed by Matthias

Commit Message

filled-date shouldn't update again

Run Details

1 of 1 new or added line in 1 file covered. (100.0%)

17024 of 17946 relevant lines covered (94.86%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.9

/freqtrade/commands/data_commands.py

import logging
import sys
from collections import defaultdict
from datetime import datetime, timedelta
from typing import Any, Dict, List

from freqtrade.configuration import TimeRange, setup_utils_configuration
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,
                                    refresh_backtest_trades_data)
from freqtrade.enums import CandleType, RunMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import market_is_active, timeframe_to_minutes
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist
from freqtrade.resolvers import ExchangeResolver
from freqtrade.util.binance_mig import migrate_binance_futures_data


logger = logging.getLogger(__name__)


def start_download_data(args: Dict[str, Any]) -> None:
    """
    Download data (former download_backtest_data.py script)
    """
    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)

    if 'days' in config and 'timerange' in config:
        raise OperationalException("--days and --timerange are mutually exclusive. "
                                   "You can only specify one or the other.")
    timerange = TimeRange()
    if 'days' in config:
        time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")
        timerange = TimeRange.parse_timerange(f'{time_since}-')

    if 'timerange' in config:
        timerange = timerange.parse_timerange(config['timerange'])

    # Remove stake-currency to skip checks which are not relevant for datadownload
    config['stake_currency'] = ''

    if 'pairs' not in config:
        raise OperationalException(
            "Downloading data requires a list of pairs. "
            "Please check the documentation on how to configure this.")

    pairs_not_available: List[str] = []

    # Init exchange
    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
    markets = [p for p, m in exchange.markets.items() if market_is_active(m)
               or config.get('include_inactive')]

    expanded_pairs = dynamic_expand_pairlist(config, markets)

    # Manual validations of relevant settings
    if not config['exchange'].get('skip_pair_validation', False):
        exchange.validate_pairs(expanded_pairs)
    logger.info(f"About to download pairs: {expanded_pairs}, "
                f"intervals: {config['timeframes']} to {config['datadir']}")

    for timeframe in config['timeframes']:
        exchange.validate_timeframes(timeframe)

    try:

        if config.get('download_trades'):
            if config.get('trading_mode') == 'futures':
                raise OperationalException("Trade download not supported for futures.")
            pairs_not_available = refresh_backtest_trades_data(
                exchange, pairs=expanded_pairs, datadir=config['datadir'],
                timerange=timerange, new_pairs_days=config['new_pairs_days'],
                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])

            # Convert downloaded trade data to different timeframes
            convert_trades_to_ohlcv(
                pairs=expanded_pairs, timeframes=config['timeframes'],
                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
                data_format_ohlcv=config['dataformat_ohlcv'],
                data_format_trades=config['dataformat_trades'],
            )
        else:
            if not exchange.get_option('ohlcv_has_history', True):
                raise OperationalException(
                    f"Historic klines not available for {exchange.name}. "
                    "Please use `--dl-trades` instead for this exchange "
                    "(will unfortunately take a long time)."
                    )
            migrate_binance_futures_data(config)
            pairs_not_available = refresh_backtest_ohlcv_data(
                exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
                datadir=config['datadir'], timerange=timerange,
                new_pairs_days=config['new_pairs_days'],
                erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
                trading_mode=config.get('trading_mode', 'spot'),
                prepend=config.get('prepend_data', False)
            )

    except KeyboardInterrupt:
        sys.exit("SIGINT received, aborting ...")

    finally:
        if pairs_not_available:
            logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "
                        f"on exchange {exchange.name}.")


def start_convert_trades(args: Dict[str, Any]) -> None:

    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)

    timerange = TimeRange()

    # Remove stake-currency to skip checks which are not relevant for datadownload
    config['stake_currency'] = ''

    if 'pairs' not in config:
        raise OperationalException(
            "Downloading data requires a list of pairs. "
            "Please check the documentation on how to configure this.")

    # Init exchange
    exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)
    # Manual validations of relevant settings
    if not config['exchange'].get('skip_pair_validation', False):
        exchange.validate_pairs(config['pairs'])
    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))

    logger.info(f"About to Convert pairs: {expanded_pairs}, "
                f"intervals: {config['timeframes']} to {config['datadir']}")

    for timeframe in config['timeframes']:
        exchange.validate_timeframes(timeframe)
    # Convert downloaded trade data to different timeframes
    convert_trades_to_ohlcv(
        pairs=expanded_pairs, timeframes=config['timeframes'],
        datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
        data_format_ohlcv=config['dataformat_ohlcv'],
        data_format_trades=config['dataformat_trades'],
    )


def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
    """
    Convert data from one format to another
    """
    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
    if ohlcv:
        migrate_binance_futures_data(config)
        candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]
        for candle_type in candle_types:
            convert_ohlcv_format(config,
                                 convert_from=args['format_from'], convert_to=args['format_to'],
                                 erase=args['erase'], candle_type=candle_type)
    else:
        convert_trades_format(config,
                              convert_from=args['format_from'], convert_to=args['format_to'],
                              erase=args['erase'])


def start_list_data(args: Dict[str, Any]) -> None:
    """
    List available backtest data
    """

    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)

    from tabulate import tabulate

    from freqtrade.data.history.idatahandler import get_datahandler
    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])

    paircombs = dhc.ohlcv_get_available_data(
        config['datadir'],
        config.get('trading_mode', TradingMode.SPOT)
        )

    if args['pairs']:
        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]

    print(f"Found {len(paircombs)} pair / timeframe combinations.")
    if not config.get('show_timerange'):
        groupedpair = defaultdict(list)
        for pair, timeframe, candle_type in sorted(
            paircombs,
            key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
        ):
            groupedpair[(pair, candle_type)].append(timeframe)

        if groupedpair:
            print(tabulate([
                (pair, ', '.join(timeframes), candle_type)
                for (pair, candle_type), timeframes in groupedpair.items()
            ],
                headers=("Pair", "Timeframe", "Type"),
                tablefmt='psql', stralign='right'))
    else:
        paircombs1 = [(
            pair, timeframe, candle_type,
            *dhc.ohlcv_data_min_max(pair, timeframe, candle_type)
        ) for pair, timeframe, candle_type in paircombs]
        print(tabulate([
            (pair, timeframe, candle_type,
                start.strftime(DATETIME_PRINT_FORMAT),
                end.strftime(DATETIME_PRINT_FORMAT))
            for pair, timeframe, candle_type, start, end in paircombs1
            ],
            headers=("Pair", "Timeframe", "Type", 'From', 'To'),
            tablefmt='psql', stralign='right'))

1	import logging	1✔
2	import sys	1✔
3	from collections import defaultdict	1✔
4	from datetime import datetime, timedelta	1✔
5	from typing import Any, Dict, List	1✔
6
7	from freqtrade.configuration import TimeRange, setup_utils_configuration	1✔
8	from freqtrade.constants import DATETIME_PRINT_FORMAT	1✔
9	from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format	1✔
10	from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data,	1✔
11	refresh_backtest_trades_data)
12	from freqtrade.enums import CandleType, RunMode, TradingMode	1✔
13	from freqtrade.exceptions import OperationalException	1✔
14	from freqtrade.exchange import market_is_active, timeframe_to_minutes	1✔
15	from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist, expand_pairlist	1✔
16	from freqtrade.resolvers import ExchangeResolver	1✔
17	from freqtrade.util.binance_mig import migrate_binance_futures_data	1✔
18
19
20	logger = logging.getLogger(__name__)	1✔
21
22
23	def start_download_data(args: Dict[str, Any]) -> None:	1✔
24	"""
25	Download data (former download_backtest_data.py script)
26	"""
27	config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)	1✔
28
29	if 'days' in config and 'timerange' in config:	1✔
30	raise OperationalException("--days and --timerange are mutually exclusive. "	1✔
31	"You can only specify one or the other.")
32	timerange = TimeRange()	1✔
33	if 'days' in config:	1✔
34	time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d")	1✔
35	timerange = TimeRange.parse_timerange(f'{time_since}-')	1✔
36
37	if 'timerange' in config:	1✔
38	timerange = timerange.parse_timerange(config['timerange'])	1✔
39
40	# Remove stake-currency to skip checks which are not relevant for datadownload
41	config['stake_currency'] = ''	1✔
42
43	if 'pairs' not in config:	1✔
44	raise OperationalException(	1✔
45	"Downloading data requires a list of pairs. "
46	"Please check the documentation on how to configure this.")
47
48	pairs_not_available: List[str] = []	1✔
49
50	# Init exchange
51	exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)	1✔
52	markets = [p for p, m in exchange.markets.items() if market_is_active(m)	1✔
53	or config.get('include_inactive')]
54
55	expanded_pairs = dynamic_expand_pairlist(config, markets)	1✔
56
57	# Manual validations of relevant settings
58	if not config['exchange'].get('skip_pair_validation', False):	1✔
59	exchange.validate_pairs(expanded_pairs)	1✔
60	logger.info(f"About to download pairs: {expanded_pairs}, "	1✔
61	f"intervals: {config['timeframes']} to {config['datadir']}")
62
63	for timeframe in config['timeframes']:	1✔
64	exchange.validate_timeframes(timeframe)	1✔
65
66	try:	1✔
67
68	if config.get('download_trades'):	1✔
69	if config.get('trading_mode') == 'futures':	1✔
70	raise OperationalException("Trade download not supported for futures.")	1✔
71	pairs_not_available = refresh_backtest_trades_data(	1✔
72	exchange, pairs=expanded_pairs, datadir=config['datadir'],
73	timerange=timerange, new_pairs_days=config['new_pairs_days'],
74	erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
75
76	# Convert downloaded trade data to different timeframes
77	convert_trades_to_ohlcv(	1✔
78	pairs=expanded_pairs, timeframes=config['timeframes'],
79	datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
80	data_format_ohlcv=config['dataformat_ohlcv'],
81	data_format_trades=config['dataformat_trades'],
82	)
83	else:
84	if not exchange.get_option('ohlcv_has_history', True):	1✔
85	raise OperationalException(	1✔
86	f"Historic klines not available for {exchange.name}. "
87	"Please use `--dl-trades` instead for this exchange "
88	"(will unfortunately take a long time)."
89	)
90	migrate_binance_futures_data(config)	1✔
91	pairs_not_available = refresh_backtest_ohlcv_data(	1✔
92	exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
93	datadir=config['datadir'], timerange=timerange,
94	new_pairs_days=config['new_pairs_days'],
95	erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'],
96	trading_mode=config.get('trading_mode', 'spot'),
97	prepend=config.get('prepend_data', False)
98	)
99
100	except KeyboardInterrupt:	1✔
101	sys.exit("SIGINT received, aborting ...")	1✔
102
103	finally:
104	if pairs_not_available:	1✔
105	logger.info(f"Pairs [{','.join(pairs_not_available)}] not available "	1✔
106	f"on exchange {exchange.name}.")
107
108
109	def start_convert_trades(args: Dict[str, Any]) -> None:	1✔
110
111	config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)	1✔
112
113	timerange = TimeRange()	1✔
114
115	# Remove stake-currency to skip checks which are not relevant for datadownload
116	config['stake_currency'] = ''	1✔
117
118	if 'pairs' not in config:	1✔
119	raise OperationalException(	×
120	"Downloading data requires a list of pairs. "
121	"Please check the documentation on how to configure this.")
122
123	# Init exchange
124	exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False)	1✔
125	# Manual validations of relevant settings
126	if not config['exchange'].get('skip_pair_validation', False):	1✔
127	exchange.validate_pairs(config['pairs'])	1✔
128	expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))	1✔
129
130	logger.info(f"About to Convert pairs: {expanded_pairs}, "	1✔
131	f"intervals: {config['timeframes']} to {config['datadir']}")
132
133	for timeframe in config['timeframes']:	1✔
134	exchange.validate_timeframes(timeframe)	1✔
135	# Convert downloaded trade data to different timeframes
136	convert_trades_to_ohlcv(	1✔
137	pairs=expanded_pairs, timeframes=config['timeframes'],
138	datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
139	data_format_ohlcv=config['dataformat_ohlcv'],
140	data_format_trades=config['dataformat_trades'],
141	)
142
143
144	def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:	1✔
145	"""
146	Convert data from one format to another
147	"""
148	config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)	1✔
149	if ohlcv:	1✔
150	migrate_binance_futures_data(config)	1✔
151	candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])]	1✔
152	for candle_type in candle_types:	1✔
153	convert_ohlcv_format(config,	1✔
154	convert_from=args['format_from'], convert_to=args['format_to'],
155	erase=args['erase'], candle_type=candle_type)
156	else:
157	convert_trades_format(config,	1✔
158	convert_from=args['format_from'], convert_to=args['format_to'],
159	erase=args['erase'])
160
161
162	def start_list_data(args: Dict[str, Any]) -> None:	1✔
163	"""
164	List available backtest data
165	"""
166
167	config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)	1✔
168
169	from tabulate import tabulate	1✔
170
171	from freqtrade.data.history.idatahandler import get_datahandler	1✔
172	dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])	1✔
173
174	paircombs = dhc.ohlcv_get_available_data(	1✔
175	config['datadir'],
176	config.get('trading_mode', TradingMode.SPOT)
177	)
178
179	if args['pairs']:	1✔
180	paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]	1✔
181
182	print(f"Found {len(paircombs)} pair / timeframe combinations.")	1✔
183	if not config.get('show_timerange'):	1✔
184	groupedpair = defaultdict(list)	1✔
185	for pair, timeframe, candle_type in sorted(	1✔
186	paircombs,
187	key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
188	):
189	groupedpair[(pair, candle_type)].append(timeframe)	1✔
190
191	if groupedpair:	1✔
192	print(tabulate([	1✔
193	(pair, ', '.join(timeframes), candle_type)
194	for (pair, candle_type), timeframes in groupedpair.items()
195	],
196	headers=("Pair", "Timeframe", "Type"),
197	tablefmt='psql', stralign='right'))
198	else:
199	paircombs1 = [(	1✔
200	pair, timeframe, candle_type,
201	*dhc.ohlcv_data_min_max(pair, timeframe, candle_type)
202	) for pair, timeframe, candle_type in paircombs]
203	print(tabulate([	1✔
204	(pair, timeframe, candle_type,
205	start.strftime(DATETIME_PRINT_FORMAT),
206	end.strftime(DATETIME_PRINT_FORMAT))
207	for pair, timeframe, candle_type, start, end in paircombs1
208	],
209	headers=("Pair", "Timeframe", "Type", 'From', 'To'),
210	tablefmt='psql', stralign='right'))

freqtrade / freqtrade / 4131164979

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous