9394559170

Committed 26 Apr 2024 06:36AM UTC coverage: 94.656% (-0.02%) from 94.674%

Build # 9394559170

Build Type

push

github

Committed by

xmatthias

Commit Message

Loader should be passed as kwarg for clarity

Run Details

20280 of 21425 relevant lines covered (94.66%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.15

/freqtrade/data/converter/trade_converter_kraken.py

import logging
from pathlib import Path

import pandas as pd

from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, Config
from freqtrade.data.converter.trade_converter import (trades_convert_types,
                                                      trades_df_remove_duplicates)
from freqtrade.data.history import get_datahandler
from freqtrade.enums import TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver


logger = logging.getLogger(__name__)

KRAKEN_CSV_TRADE_COLUMNS = ['timestamp', 'price', 'amount']


def import_kraken_trades_from_csv(config: Config, convert_to: str):
    """
    Import kraken trades from csv
    """
    if config['exchange']['name'] != 'kraken':
        raise OperationalException('This function is only for the kraken exchange.')

    datadir: Path = config['datadir']
    data_handler = get_datahandler(datadir, data_format=convert_to)

    tradesdir: Path = config['datadir'] / 'trades_csv'
    exchange = ExchangeResolver.load_exchange(config, validate=False)
    # iterate through directories in this directory
    data_symbols = {p.stem for p in tradesdir.rglob('*.csv')}

    # create pair/filename mapping
    markets = {
        (m['symbol'], m['altname']) for m in exchange.markets.values()
        if m.get('altname') in data_symbols
    }
    logger.info(f"Found csv files for {', '.join(data_symbols)}.")

    if pairs_raw := config.get('pairs'):
        pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])
        markets = {m for m in markets if m[0] in pairs}
        if not markets:
            logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")
            return
    logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")

    for pair, name in markets:
        logger.debug(f"Converting pair {pair}, files */{name}.csv")
        dfs = []
        # Load and combine all csv files for this pair
        for f in tradesdir.rglob(f"{name}.csv"):
            df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS)
            if not df.empty:
                dfs.append(df)

        # Load existing trades data
        if not dfs:
            # edgecase, can only happen if the file was deleted between the above glob and here
            logger.info(f"No data found for pair {pair}")
            continue

        trades = pd.concat(dfs, ignore_index=True)
        del dfs

        trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3
        trades.loc[:, 'cost'] = trades['price'] * trades['amount']
        for col in DEFAULT_TRADES_COLUMNS:
            if col not in trades.columns:
                trades.loc[:, col] = ''
        trades = trades[DEFAULT_TRADES_COLUMNS]
        trades = trades_convert_types(trades)

        trades_df = trades_df_remove_duplicates(trades)
        del trades
        logger.info(f"{pair}: {len(trades_df)} trades, from "
                    f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
                    f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")

        data_handler.trades_store(pair, trades_df, TradingMode.SPOT)

1	import logging	1✔
2	from pathlib import Path	1✔
3
4	import pandas as pd	1✔
5
6	from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, Config	1✔
7	from freqtrade.data.converter.trade_converter import (trades_convert_types,	1✔
8	trades_df_remove_duplicates)
9	from freqtrade.data.history import get_datahandler	1✔
10	from freqtrade.enums import TradingMode	1✔
11	from freqtrade.exceptions import OperationalException	1✔
12	from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist	1✔
13	from freqtrade.resolvers import ExchangeResolver	1✔
14
15
16	logger = logging.getLogger(__name__)	1✔
17
18	KRAKEN_CSV_TRADE_COLUMNS = ['timestamp', 'price', 'amount']	1✔
19
20
21	def import_kraken_trades_from_csv(config: Config, convert_to: str):	1✔
22	"""
23	Import kraken trades from csv
24	"""
25	if config['exchange']['name'] != 'kraken':	1✔
26	raise OperationalException('This function is only for the kraken exchange.')	1✔
27
28	datadir: Path = config['datadir']	1✔
29	data_handler = get_datahandler(datadir, data_format=convert_to)	1✔
30
31	tradesdir: Path = config['datadir'] / 'trades_csv'	1✔
32	exchange = ExchangeResolver.load_exchange(config, validate=False)	1✔
33	# iterate through directories in this directory
34	data_symbols = {p.stem for p in tradesdir.rglob('*.csv')}	1✔
35
36	# create pair/filename mapping
37	markets = {	1✔
38	(m['symbol'], m['altname']) for m in exchange.markets.values()
39	if m.get('altname') in data_symbols
40	}
41	logger.info(f"Found csv files for {', '.join(data_symbols)}.")	1✔
42
43	if pairs_raw := config.get('pairs'):	1✔
44	pairs = expand_pairlist(pairs_raw, [m[0] for m in markets])	1✔
45	markets = {m for m in markets if m[0] in pairs}	1✔
46	if not markets:	1✔
47	logger.info(f"No data found for pairs {', '.join(pairs_raw)}.")	1✔
48	return	1✔
49	logger.info(f"Converting pairs: {', '.join(m[0] for m in markets)}.")	1✔
50
51	for pair, name in markets:	1✔
52	logger.debug(f"Converting pair {pair}, files */{name}.csv")	1✔
53	dfs = []	1✔
54	# Load and combine all csv files for this pair
55	for f in tradesdir.rglob(f"{name}.csv"):	1✔
56	df = pd.read_csv(f, names=KRAKEN_CSV_TRADE_COLUMNS)	1✔
57	if not df.empty:	1✔
58	dfs.append(df)	1✔
59
60	# Load existing trades data
61	if not dfs:	1✔
62	# edgecase, can only happen if the file was deleted between the above glob and here
63	logger.info(f"No data found for pair {pair}")	×
64	continue	×
65
66	trades = pd.concat(dfs, ignore_index=True)	1✔
67	del dfs	1✔
68
69	trades.loc[:, 'timestamp'] = trades['timestamp'] * 1e3	1✔
70	trades.loc[:, 'cost'] = trades['price'] * trades['amount']	1✔
71	for col in DEFAULT_TRADES_COLUMNS:	1✔
72	if col not in trades.columns:	1✔
73	trades.loc[:, col] = ''	1✔
74	trades = trades[DEFAULT_TRADES_COLUMNS]	1✔
75	trades = trades_convert_types(trades)	1✔
76
77	trades_df = trades_df_remove_duplicates(trades)	1✔
78	del trades	1✔
79	logger.info(f"{pair}: {len(trades_df)} trades, from "	1✔
80	f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
81	f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")
82
83	data_handler.trades_store(pair, trades_df, TradingMode.SPOT)	1✔

freqtrade / freqtrade / 9394559170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous