6181253459

Committed 08 Sep 2023 06:04AM UTC coverage: 94.614% (+0.06%) from 94.556%

Build # 6181253459

Build Type

push

github-actions

Committed by

web-flow

Commit Message

Merge pull request #9159 from stash86/fix-adjust

remove old codes when we only can do partial entries

Run Details

2 of 2 new or added lines in 1 file covered. (100.0%)

19114 of 20202 relevant lines covered (94.61%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.48

/freqtrade/commands/data_commands.py

import logging
import sys
from collections import defaultdict
from typing import Any, Dict

from freqtrade.configuration import TimeRange, setup_utils_configuration
from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config
from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format
from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main
from freqtrade.enums import RunMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver
from freqtrade.util.binance_mig import migrate_binance_futures_data


logger = logging.getLogger(__name__)


def _check_data_config_download_sanity(config: Config) -> None:
    if 'days' in config and 'timerange' in config:
        raise OperationalException("--days and --timerange are mutually exclusive. "
                                   "You can only specify one or the other.")

    if 'pairs' not in config:
        raise OperationalException(
            "Downloading data requires a list of pairs. "
            "Please check the documentation on how to configure this.")


def start_download_data(args: Dict[str, Any]) -> None:
    """
    Download data (former download_backtest_data.py script)
    """
    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)

    _check_data_config_download_sanity(config)

    try:
        download_data_main(config)

    except KeyboardInterrupt:
        sys.exit("SIGINT received, aborting ...")


def start_convert_trades(args: Dict[str, Any]) -> None:

    config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)

    timerange = TimeRange()

    # Remove stake-currency to skip checks which are not relevant for datadownload
    config['stake_currency'] = ''

    if 'pairs' not in config:
        raise OperationalException(
            "Downloading data requires a list of pairs. "
            "Please check the documentation on how to configure this.")
    if 'timeframes' not in config:
        config['timeframes'] = DL_DATA_TIMEFRAMES

    # Init exchange
    exchange = ExchangeResolver.load_exchange(config, validate=False)
    # Manual validations of relevant settings
    if not config['exchange'].get('skip_pair_validation', False):
        exchange.validate_pairs(config['pairs'])
    expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))

    logger.info(f"About to Convert pairs: {expanded_pairs}, "
                f"intervals: {config['timeframes']} to {config['datadir']}")

    for timeframe in config['timeframes']:
        exchange.validate_timeframes(timeframe)
    # Convert downloaded trade data to different timeframes
    convert_trades_to_ohlcv(
        pairs=expanded_pairs, timeframes=config['timeframes'],
        datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
        data_format_ohlcv=config['dataformat_ohlcv'],
        data_format_trades=config['dataformat_trades'],
    )


def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:
    """
    Convert data from one format to another
    """
    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
    if ohlcv:
        migrate_binance_futures_data(config)
        convert_ohlcv_format(config,
                             convert_from=args['format_from'],
                             convert_to=args['format_to'],
                             erase=args['erase'])
    else:
        convert_trades_format(config,
                              convert_from=args['format_from'], convert_to=args['format_to'],
                              erase=args['erase'])


def start_list_data(args: Dict[str, Any]) -> None:
    """
    List available backtest data
    """

    config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)

    from tabulate import tabulate

    from freqtrade.data.history.idatahandler import get_datahandler
    dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])

    paircombs = dhc.ohlcv_get_available_data(
        config['datadir'],
        config.get('trading_mode', TradingMode.SPOT)
        )

    if args['pairs']:
        paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]

    print(f"Found {len(paircombs)} pair / timeframe combinations.")
    if not config.get('show_timerange'):
        groupedpair = defaultdict(list)
        for pair, timeframe, candle_type in sorted(
            paircombs,
            key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
        ):
            groupedpair[(pair, candle_type)].append(timeframe)

        if groupedpair:
            print(tabulate([
                (pair, ', '.join(timeframes), candle_type)
                for (pair, candle_type), timeframes in groupedpair.items()
            ],
                headers=("Pair", "Timeframe", "Type"),
                tablefmt='psql', stralign='right'))
    else:
        paircombs1 = [(
            pair, timeframe, candle_type,
            *dhc.ohlcv_data_min_max(pair, timeframe, candle_type)
        ) for pair, timeframe, candle_type in paircombs]

        print(tabulate([
            (pair, timeframe, candle_type,
                start.strftime(DATETIME_PRINT_FORMAT),
                end.strftime(DATETIME_PRINT_FORMAT))
            for pair, timeframe, candle_type, start, end in sorted(
                paircombs1,
                key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
            ],
            headers=("Pair", "Timeframe", "Type", 'From', 'To'),
            tablefmt='psql', stralign='right'))

1	import logging	1✔
2	import sys	1✔
3	from collections import defaultdict	1✔
4	from typing import Any, Dict	1✔
5
6	from freqtrade.configuration import TimeRange, setup_utils_configuration	1✔
7	from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Config	1✔
8	from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format	1✔
9	from freqtrade.data.history import convert_trades_to_ohlcv, download_data_main	1✔
10	from freqtrade.enums import RunMode, TradingMode	1✔
11	from freqtrade.exceptions import OperationalException	1✔
12	from freqtrade.exchange import timeframe_to_minutes	1✔
13	from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist	1✔
14	from freqtrade.resolvers import ExchangeResolver	1✔
15	from freqtrade.util.binance_mig import migrate_binance_futures_data	1✔
16
17
18	logger = logging.getLogger(__name__)	1✔
19
20
21	def _check_data_config_download_sanity(config: Config) -> None:	1✔
22	if 'days' in config and 'timerange' in config:	1✔
23	raise OperationalException("--days and --timerange are mutually exclusive. "	1✔
24	"You can only specify one or the other.")
25
26	if 'pairs' not in config:	1✔
27	raise OperationalException(	1✔
28	"Downloading data requires a list of pairs. "
29	"Please check the documentation on how to configure this.")
30
31
32	def start_download_data(args: Dict[str, Any]) -> None:	1✔
33	"""
34	Download data (former download_backtest_data.py script)
35	"""
36	config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)	1✔
37
38	_check_data_config_download_sanity(config)	1✔
39
40	try:	1✔
41	download_data_main(config)	1✔
42
43	except KeyboardInterrupt:	1✔
44	sys.exit("SIGINT received, aborting ...")	1✔
45
46
47	def start_convert_trades(args: Dict[str, Any]) -> None:	1✔
48
49	config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)	1✔
50
51	timerange = TimeRange()	1✔
52
53	# Remove stake-currency to skip checks which are not relevant for datadownload
54	config['stake_currency'] = ''	1✔
55
56	if 'pairs' not in config:	1✔
57	raise OperationalException(	×
58	"Downloading data requires a list of pairs. "
59	"Please check the documentation on how to configure this.")
60	if 'timeframes' not in config:	1✔
61	config['timeframes'] = DL_DATA_TIMEFRAMES	1✔
62
63	# Init exchange
64	exchange = ExchangeResolver.load_exchange(config, validate=False)	1✔
65	# Manual validations of relevant settings
66	if not config['exchange'].get('skip_pair_validation', False):	1✔
67	exchange.validate_pairs(config['pairs'])	1✔
68	expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets))	1✔
69
70	logger.info(f"About to Convert pairs: {expanded_pairs}, "	1✔
71	f"intervals: {config['timeframes']} to {config['datadir']}")
72
73	for timeframe in config['timeframes']:	1✔
74	exchange.validate_timeframes(timeframe)	1✔
75	# Convert downloaded trade data to different timeframes
76	convert_trades_to_ohlcv(	1✔
77	pairs=expanded_pairs, timeframes=config['timeframes'],
78	datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
79	data_format_ohlcv=config['dataformat_ohlcv'],
80	data_format_trades=config['dataformat_trades'],
81	)
82
83
84	def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None:	1✔
85	"""
86	Convert data from one format to another
87	"""
88	config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)	1✔
89	if ohlcv:	1✔
90	migrate_binance_futures_data(config)	1✔
91	convert_ohlcv_format(config,	1✔
92	convert_from=args['format_from'],
93	convert_to=args['format_to'],
94	erase=args['erase'])
95	else:
96	convert_trades_format(config,	1✔
97	convert_from=args['format_from'], convert_to=args['format_to'],
98	erase=args['erase'])
99
100
101	def start_list_data(args: Dict[str, Any]) -> None:	1✔
102	"""
103	List available backtest data
104	"""
105
106	config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)	1✔
107
108	from tabulate import tabulate	1✔
109
110	from freqtrade.data.history.idatahandler import get_datahandler	1✔
111	dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])	1✔
112
113	paircombs = dhc.ohlcv_get_available_data(	1✔
114	config['datadir'],
115	config.get('trading_mode', TradingMode.SPOT)
116	)
117
118	if args['pairs']:	1✔
119	paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]	1✔
120
121	print(f"Found {len(paircombs)} pair / timeframe combinations.")	1✔
122	if not config.get('show_timerange'):	1✔
123	groupedpair = defaultdict(list)	1✔
124	for pair, timeframe, candle_type in sorted(	1✔
125	paircombs,
126	key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2])
127	):
128	groupedpair[(pair, candle_type)].append(timeframe)	1✔
129
130	if groupedpair:	1✔
131	print(tabulate([	1✔
132	(pair, ', '.join(timeframes), candle_type)
133	for (pair, candle_type), timeframes in groupedpair.items()
134	],
135	headers=("Pair", "Timeframe", "Type"),
136	tablefmt='psql', stralign='right'))
137	else:
138	paircombs1 = [(	1✔
139	pair, timeframe, candle_type,
140	*dhc.ohlcv_data_min_max(pair, timeframe, candle_type)
141	) for pair, timeframe, candle_type in paircombs]
142
143	print(tabulate([	1✔
144	(pair, timeframe, candle_type,
145	start.strftime(DATETIME_PRINT_FORMAT),
146	end.strftime(DATETIME_PRINT_FORMAT))
147	for pair, timeframe, candle_type, start, end in sorted(
148	paircombs1,
149	key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]))
150	],
151	headers=("Pair", "Timeframe", "Type", 'From', 'To'),
152	tablefmt='psql', stralign='right'))

freqtrade / freqtrade / 6181253459

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous