9394559170

Committed 26 Apr 2024 06:36AM UTC coverage: 94.656% (-0.02%) from 94.674%

Build # 9394559170

Build Type

push

github

Committed by

xmatthias

Commit Message

Loader should be passed as kwarg for clarity

Run Details

20280 of 21425 relevant lines covered (94.66%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.51

/freqtrade/data/history/datahandlers/hdf5datahandler.py

import logging
from typing import Optional

import numpy as np
import pandas as pd

from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.enums import CandleType, TradingMode

from .idatahandler import IDataHandler


logger = logging.getLogger(__name__)


class HDF5DataHandler(IDataHandler):

    _columns = DEFAULT_DATAFRAME_COLUMNS

    def ohlcv_store(
            self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType) -> None:
        """
        Store data in hdf5 file.
        :param pair: Pair - used to generate filename
        :param timeframe: Timeframe - used to generate filename
        :param data: Dataframe containing OHLCV data
        :param candle_type: Any of the enum CandleType (must match trading mode!)
        :return: None
        """
        key = self._pair_ohlcv_key(pair, timeframe)
        _data = data.copy()

        filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)
        self.create_dir_if_needed(filename)

        _data.loc[:, self._columns].to_hdf(
            filename, key=key, mode='a', complevel=9, complib='blosc',
            format='table', data_columns=['date']
        )

    def _ohlcv_load(self, pair: str, timeframe: str,
                    timerange: Optional[TimeRange], candle_type: CandleType
                    ) -> pd.DataFrame:
        """
        Internal method used to load data for one pair from disk.
        Implements the loading and conversion to a Pandas dataframe.
        Timerange trimming and dataframe validation happens outside of this method.
        :param pair: Pair to load data
        :param timeframe: Timeframe (e.g. "5m")
        :param timerange: Limit data to be loaded to this timerange.
                        Optionally implemented by subclasses to avoid loading
                        all data where possible.
        :param candle_type: Any of the enum CandleType (must match trading mode!)
        :return: DataFrame with ohlcv data, or empty DataFrame
        """
        key = self._pair_ohlcv_key(pair, timeframe)
        filename = self._pair_data_filename(
            self._datadir,
            pair,
            timeframe,
            candle_type=candle_type
        )

        if not filename.exists():
            # Fallback mode for 1M files
            filename = self._pair_data_filename(
                self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
            if not filename.exists():
                return pd.DataFrame(columns=self._columns)
        where = []
        if timerange:
            if timerange.starttype == 'date':
                where.append(f"date >= Timestamp({timerange.startts * 1e9})")
            if timerange.stoptype == 'date':
                where.append(f"date <= Timestamp({timerange.stopts * 1e9})")

        pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)

        if list(pairdata.columns) != self._columns:
            raise ValueError("Wrong dataframe format")
        pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',
                                          'low': 'float', 'close': 'float', 'volume': 'float'})
        pairdata = pairdata.reset_index(drop=True)
        return pairdata

    def ohlcv_append(
        self,
        pair: str,
        timeframe: str,
        data: pd.DataFrame,
        candle_type: CandleType
    ) -> None:
        """
        Append data to existing data structures
        :param pair: Pair
        :param timeframe: Timeframe this ohlcv data is for
        :param data: Data to append.
        :param candle_type: Any of the enum CandleType (must match trading mode!)
        """
        raise NotImplementedError()

    def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None:
        """
        Store trades data (list of Dicts) to file
        :param pair: Pair - used for filename
        :param data: Dataframe containing trades
                     column sequence as in DEFAULT_TRADES_COLUMNS
        :param trading_mode: Trading mode to use (used to determine the filename)
        """
        key = self._pair_trades_key(pair)

        data.to_hdf(
            self._pair_trades_filename(self._datadir, pair, trading_mode), key=key,
            mode='a', complevel=9, complib='blosc',
            format='table', data_columns=['timestamp']
        )

    def trades_append(self, pair: str, data: pd.DataFrame):
        """
        Append data to existing files
        :param pair: Pair - used for filename
        :param data: Dataframe containing trades
                     column sequence as in DEFAULT_TRADES_COLUMNS
        """
        raise NotImplementedError()

    def _trades_load(
        self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
    ) -> pd.DataFrame:
        """
        Load a pair from h5 file.
        :param pair: Load trades for this pair
        :param trading_mode: Trading mode to use (used to determine the filename)
        :param timerange: Timerange to load trades for - currently not implemented
        :return: Dataframe containing trades
        """
        key = self._pair_trades_key(pair)
        filename = self._pair_trades_filename(self._datadir, pair, trading_mode)

        if not filename.exists():
            return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)
        where = []
        if timerange:
            if timerange.starttype == 'date':
                where.append(f"timestamp >= {timerange.startts * 1e3}")
            if timerange.stoptype == 'date':
                where.append(f"timestamp < {timerange.stopts * 1e3}")

        trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where)
        trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})
        return trades

    @classmethod
    def _get_file_extension(cls):
        return "h5"

    @classmethod
    def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:
        # Escape futures pairs to avoid warnings
        pair_esc = pair.replace(':', '_')
        return f"{pair_esc}/ohlcv/tf_{timeframe}"

    @classmethod
    def _pair_trades_key(cls, pair: str) -> str:
        return f"{pair}/trades"

1	import logging	1✔
2	from typing import Optional	1✔
3
4	import numpy as np	1✔
5	import pandas as pd	1✔
6
7	from freqtrade.configuration import TimeRange	1✔
8	from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS	1✔
9	from freqtrade.enums import CandleType, TradingMode	1✔
10
11	from .idatahandler import IDataHandler	1✔
12
13
14	logger = logging.getLogger(__name__)	1✔
15
16
17	class HDF5DataHandler(IDataHandler):	1✔
18
19	_columns = DEFAULT_DATAFRAME_COLUMNS	1✔
20
21	def ohlcv_store(	1✔
22	self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType) -> None:
23	"""
24	Store data in hdf5 file.
25	:param pair: Pair - used to generate filename
26	:param timeframe: Timeframe - used to generate filename
27	:param data: Dataframe containing OHLCV data
28	:param candle_type: Any of the enum CandleType (must match trading mode!)
29	:return: None
30	"""
31	key = self._pair_ohlcv_key(pair, timeframe)	1✔
32	_data = data.copy()	1✔
33
34	filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type)	1✔
35	self.create_dir_if_needed(filename)	1✔
36
37	_data.loc[:, self._columns].to_hdf(	1✔
38	filename, key=key, mode='a', complevel=9, complib='blosc',
39	format='table', data_columns=['date']
40	)
41
42	def _ohlcv_load(self, pair: str, timeframe: str,	1✔
43	timerange: Optional[TimeRange], candle_type: CandleType
44	) -> pd.DataFrame:
45	"""
46	Internal method used to load data for one pair from disk.
47	Implements the loading and conversion to a Pandas dataframe.
48	Timerange trimming and dataframe validation happens outside of this method.
49	:param pair: Pair to load data
50	:param timeframe: Timeframe (e.g. "5m")
51	:param timerange: Limit data to be loaded to this timerange.
52	Optionally implemented by subclasses to avoid loading
53	all data where possible.
54	:param candle_type: Any of the enum CandleType (must match trading mode!)
55	:return: DataFrame with ohlcv data, or empty DataFrame
56	"""
57	key = self._pair_ohlcv_key(pair, timeframe)	1✔
58	filename = self._pair_data_filename(	1✔
59	self._datadir,
60	pair,
61	timeframe,
62	candle_type=candle_type
63	)
64
65	if not filename.exists():	1✔
66	# Fallback mode for 1M files
67	filename = self._pair_data_filename(	1✔
68	self._datadir, pair, timeframe, candle_type=candle_type, no_timeframe_modify=True)
69	if not filename.exists():	1✔
70	return pd.DataFrame(columns=self._columns)	1✔
71	where = []	1✔
72	if timerange:	1✔
73	if timerange.starttype == 'date':	1✔
74	where.append(f"date >= Timestamp({timerange.startts * 1e9})")	1✔
75	if timerange.stoptype == 'date':	1✔
76	where.append(f"date <= Timestamp({timerange.stopts * 1e9})")	1✔
77
78	pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)	1✔
79
80	if list(pairdata.columns) != self._columns:	1✔
81	raise ValueError("Wrong dataframe format")	×
82	pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float',	1✔
83	'low': 'float', 'close': 'float', 'volume': 'float'})
84	pairdata = pairdata.reset_index(drop=True)	1✔
85	return pairdata	1✔
86
87	def ohlcv_append(	1✔
88	self,
89	pair: str,
90	timeframe: str,
91	data: pd.DataFrame,
92	candle_type: CandleType
93	) -> None:
94	"""
95	Append data to existing data structures
96	:param pair: Pair
97	:param timeframe: Timeframe this ohlcv data is for
98	:param data: Data to append.
99	:param candle_type: Any of the enum CandleType (must match trading mode!)
100	"""
101	raise NotImplementedError()	1✔
102
103	def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None:	1✔
104	"""
105	Store trades data (list of Dicts) to file
106	:param pair: Pair - used for filename
107	:param data: Dataframe containing trades
108	column sequence as in DEFAULT_TRADES_COLUMNS
109	:param trading_mode: Trading mode to use (used to determine the filename)
110	"""
111	key = self._pair_trades_key(pair)	1✔
112
113	data.to_hdf(	1✔
114	self._pair_trades_filename(self._datadir, pair, trading_mode), key=key,
115	mode='a', complevel=9, complib='blosc',
116	format='table', data_columns=['timestamp']
117	)
118
119	def trades_append(self, pair: str, data: pd.DataFrame):	1✔
120	"""
121	Append data to existing files
122	:param pair: Pair - used for filename
123	:param data: Dataframe containing trades
124	column sequence as in DEFAULT_TRADES_COLUMNS
125	"""
126	raise NotImplementedError()	1✔
127
128	def _trades_load(	1✔
129	self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
130	) -> pd.DataFrame:
131	"""
132	Load a pair from h5 file.
133	:param pair: Load trades for this pair
134	:param trading_mode: Trading mode to use (used to determine the filename)
135	:param timerange: Timerange to load trades for - currently not implemented
136	:return: Dataframe containing trades
137	"""
138	key = self._pair_trades_key(pair)	1✔
139	filename = self._pair_trades_filename(self._datadir, pair, trading_mode)	1✔
140
141	if not filename.exists():	1✔
142	return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)	1✔
143	where = []	1✔
144	if timerange:	1✔
145	if timerange.starttype == 'date':	1✔
146	where.append(f"timestamp >= {timerange.startts * 1e3}")	1✔
147	if timerange.stoptype == 'date':	1✔
148	where.append(f"timestamp < {timerange.stopts * 1e3}")	1✔
149
150	trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where)	1✔
151	trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None})	1✔
152	return trades	1✔
153
154	@classmethod	1✔
155	def _get_file_extension(cls):	1✔
156	return "h5"	1✔
157
158	@classmethod	1✔
159	def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str:	1✔
160	# Escape futures pairs to avoid warnings
161	pair_esc = pair.replace(':', '_')	1✔
162	return f"{pair_esc}/ohlcv/tf_{timeframe}"	1✔
163
164	@classmethod	1✔
165	def _pair_trades_key(cls, pair: str) -> str:	1✔
166	return f"{pair}/trades"	1✔

freqtrade / freqtrade / 9394559170

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous