1557291546

Build Type

push

github-actions

Committed by Matthias

Commit Message

Fix delete_Trade api test

Run Details

10257 of 10439 relevant lines covered (98.26%)

0.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.18

/freqtrade/edge/edge_positioning.py

# pragma pylint: disable=W0603
""" Edge positioning package """
import logging
from collections import defaultdict
from copy import deepcopy
from typing import Any, Dict, List, NamedTuple

import arrow
import numpy as np
import utils_find_1st as utf1st
from pandas import DataFrame

from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.enums import RunMode, SellType
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.exchange import timeframe_to_seconds
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.strategy.interface import IStrategy


logger = logging.getLogger(__name__)


class PairInfo(NamedTuple):
    stoploss: float
    winrate: float
    risk_reward_ratio: float
    required_risk_reward: float
    expectancy: float
    nb_trades: int
    avg_trade_duration: float


class Edge:
    """
    Calculates Win Rate, Risk Reward Ratio, Expectancy
    against historical data for a give set of markets and a strategy
    it then adjusts stoploss and position size accordingly
    and force it into the strategy
    Author: https://github.com/mishaker
    """

    config: Dict = {}
    _cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs

    def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:

        self.config = config
        self.exchange = exchange
        self.strategy: IStrategy = strategy

        self.edge_config = self.config.get('edge', {})
        self._cached_pairs: Dict[str, Any] = {}  # Keeps a list of pairs
        self._final_pairs: list = []

        # checking max_open_trades. it should be -1 as with Edge
        # the number of trades is determined by position size
        if self.config['max_open_trades'] != float('inf'):
            logger.critical('max_open_trades should be -1 in config !')

        if self.config['stake_amount'] != UNLIMITED_STAKE_AMOUNT:
            raise OperationalException('Edge works only with unlimited stake amount')

        self._capital_ratio: float = self.config['tradable_balance_ratio']
        self._allowed_risk: float = self.edge_config.get('allowed_risk')
        self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14)
        self._last_updated: int = 0  # Timestamp of pairs last updated time
        self._refresh_pairs = True

        self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01))
        self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05))
        self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001))

        # calculating stoploss range
        self._stoploss_range = np.arange(
            self._stoploss_range_min,
            self._stoploss_range_max,
            self._stoploss_range_step
        )

        self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(
            days=-1 * self._since_number_of_days).format('YYYYMMDD'))
        if config.get('fee'):
            self.fee = config['fee']
        else:
            try:
                self.fee = self.exchange.get_fee(symbol=expand_pairlist(
                    self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
            except IndexError:
                self.fee = None

    def calculate(self, pairs: List[str]) -> bool:
        if self.fee is None and pairs:
            self.fee = self.exchange.get_fee(pairs[0])

        heartbeat = self.edge_config.get('process_throttle_secs')

        if (self._last_updated > 0) and (
                self._last_updated + heartbeat > arrow.utcnow().int_timestamp):
            return False

        data: Dict[str, Any] = {}
        logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
        logger.info('Using local backtesting data (using whitelist in given config) ...')

        if self._refresh_pairs:
            timerange_startup = deepcopy(self._timerange)
            timerange_startup.subtract_start(timeframe_to_seconds(
                self.strategy.timeframe) * self.strategy.startup_candle_count)
            refresh_data(
                datadir=self.config['datadir'],
                pairs=pairs,
                exchange=self.exchange,
                timeframe=self.strategy.timeframe,
                timerange=timerange_startup,
                data_format=self.config.get('dataformat_ohlcv', 'json'),
            )
            # Download informative pairs too
            res = defaultdict(list)
            for p, t in self.strategy.gather_informative_pairs():
                res[t].append(p)
            for timeframe, inf_pairs in res.items():
                timerange_startup = deepcopy(self._timerange)
                timerange_startup.subtract_start(timeframe_to_seconds(
                    timeframe) * self.strategy.startup_candle_count)
                refresh_data(
                    datadir=self.config['datadir'],
                    pairs=inf_pairs,
                    exchange=self.exchange,
                    timeframe=timeframe,
                    timerange=timerange_startup,
                    data_format=self.config.get('dataformat_ohlcv', 'json'),
                )

        data = load_data(
            datadir=self.config['datadir'],
            pairs=pairs,
            timeframe=self.strategy.timeframe,
            timerange=self._timerange,
            startup_candles=self.strategy.startup_candle_count,
            data_format=self.config.get('dataformat_ohlcv', 'json'),
        )

        if not data:
            # Reinitializing cached pairs
            self._cached_pairs = {}
            logger.critical("No data found. Edge is stopped ...")
            return False
        # Fake run-mode to Edge
        prior_rm = self.config['runmode']
        self.config['runmode'] = RunMode.EDGE
        preprocessed = self.strategy.advise_all_indicators(data)
        self.config['runmode'] = prior_rm

        # Print timeframe
        min_date, max_date = get_timerange(preprocessed)
        logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
                    f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
                    f'({(max_date - min_date).days} days)..')
        headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']

        trades: list = []
        for pair, pair_data in preprocessed.items():
            # Sorting dataframe by date and reset index
            pair_data = pair_data.sort_values(by=['date'])
            pair_data = pair_data.reset_index(drop=True)

            df_analyzed = self.strategy.advise_sell(
                self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()

            trades += self._find_trades_for_stoploss_range(df_analyzed, pair, self._stoploss_range)

        # If no trade found then exit
        if len(trades) == 0:
            logger.info("No trades found.")
            return False

        # Fill missing, calculable columns, profit, duration , abs etc.
        trades_df = self._fill_calculable_fields(DataFrame(trades))
        self._cached_pairs = self._process_expectancy(trades_df)
        self._last_updated = arrow.utcnow().int_timestamp

        return True

    def stake_amount(self, pair: str, free_capital: float,
                     total_capital: float, capital_in_trade: float) -> float:
        stoploss = self.stoploss(pair)
        available_capital = (total_capital + capital_in_trade) * self._capital_ratio
        allowed_capital_at_risk = available_capital * self._allowed_risk
        max_position_size = abs(allowed_capital_at_risk / stoploss)
        # Position size must be below available capital.
        position_size = min(min(max_position_size, free_capital), available_capital)
        if pair in self._cached_pairs:
            logger.info(
                'winrate: %s, expectancy: %s, position size: %s, pair: %s,'
                ' capital in trade: %s, free capital: %s, total capital: %s,'
                ' stoploss: %s, available capital: %s.',
                self._cached_pairs[pair].winrate,
                self._cached_pairs[pair].expectancy,
                position_size, pair,
                capital_in_trade, free_capital, total_capital,
                stoploss, available_capital
            )
        return round(position_size, 15)

    def stoploss(self, pair: str) -> float:
        if pair in self._cached_pairs:
            return self._cached_pairs[pair].stoploss
        else:
            logger.warning(f'Tried to access stoploss of non-existing pair {pair}, '
                           'strategy stoploss is returned instead.')
            return self.strategy.stoploss

    def adjust(self, pairs: List[str]) -> list:
        """
        Filters out and sorts "pairs" according to Edge calculated pairs
        """
        final = []
        for pair, info in self._cached_pairs.items():
            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
                info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
                    pair in pairs:
                final.append(pair)

        if self._final_pairs != final:
            self._final_pairs = final
            if self._final_pairs:
                logger.info(
                    'Minimum expectancy and minimum winrate are met only for %s,'
                    ' so other pairs are filtered out.',
                    self._final_pairs
                )
            else:
                logger.info(
                    'Edge removed all pairs as no pair with minimum expectancy '
                    'and minimum winrate was found !'
                )

        return self._final_pairs

    def accepted_pairs(self) -> List[Dict[str, Any]]:
        """
        return a list of accepted pairs along with their winrate, expectancy and stoploss
        """
        final = []
        for pair, info in self._cached_pairs.items():
            if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \
                    info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
                final.append({
                    'Pair': pair,
                    'Winrate': info.winrate,
                    'Expectancy': info.expectancy,
                    'Stoploss': info.stoploss,
                })
        return final

    def _fill_calculable_fields(self, result: DataFrame) -> DataFrame:
        """
        The result frame contains a number of columns that are calculable
        from other columns. These are left blank till all rows are added,
        to be populated in single vector calls.

        Columns to be populated are:
        - Profit
        - trade duration
        - profit abs
        :param result Dataframe
        :return: result Dataframe
        """
        # We set stake amount to an arbitrary amount, as it doesn't change the calculation.
        # All returned values are relative, they are defined as ratios.
        stake = 0.015

        result['trade_duration'] = result['close_date'] - result['open_date']

        result['trade_duration'] = result['trade_duration'].map(
            lambda x: int(x.total_seconds() / 60))

        # Spends, Takes, Profit, Absolute Profit

        # Buy Price
        result['buy_vol'] = stake / result['open_rate']  # How many target are we buying
        result['buy_fee'] = stake * self.fee
        result['buy_spend'] = stake + result['buy_fee']  # How much we're spending

        # Sell price
        result['sell_sum'] = result['buy_vol'] * result['close_rate']
        result['sell_fee'] = result['sell_sum'] * self.fee
        result['sell_take'] = result['sell_sum'] - result['sell_fee']

        # profit_ratio
        result['profit_ratio'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']

        # Absolute profit
        result['profit_abs'] = result['sell_take'] - result['buy_spend']

        return result

    def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:
        """
        This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
        The calculation will be done per pair and per strategy.
        """
        # Removing pairs having less than min_trades_number
        min_trades_number = self.edge_config.get('min_trade_number', 10)
        results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number)
        ###################################

        # Removing outliers (Only Pumps) from the dataset
        # The method to detect outliers is to calculate standard deviation
        # Then every value more than (standard deviation + 2*average) is out (pump)
        #
        # Removing Pumps
        if self.edge_config.get('remove_pumps', False):
            results = results[results['profit_abs'] < 2 * results['profit_abs'].std()
                              + results['profit_abs'].mean()]
        ##########################################################################

        # Removing trades having a duration more than X minutes (set in config)
        max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440)
        results = results[results.trade_duration < max_trade_duration]
        #######################################################################

        if results.empty:
            return {}

        groupby_aggregator = {
            'profit_abs': [
                ('nb_trades', 'count'),  # number of all trades
                ('profit_sum', lambda x: x[x > 0].sum()),  # cumulative profit of all winning trades
                ('loss_sum', lambda x: abs(x[x < 0].sum())),  # cumulative loss of all losing trades
                ('nb_win_trades', lambda x: x[x > 0].count())  # number of winning trades
            ],
            'trade_duration': [('avg_trade_duration', 'mean')]
        }

        # Group by (pair and stoploss) by applying above aggregator
        df = results.groupby(['pair', 'stoploss'])[['profit_abs', 'trade_duration']].agg(
            groupby_aggregator).reset_index(col_level=1)

        # Dropping level 0 as we don't need it
        df.columns = df.columns.droplevel(0)

        # Calculating number of losing trades, average win and average loss
        df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']
        df['average_win'] = np.where(df['nb_win_trades'] == 0, 0.0,
                                     df['profit_sum'] / df['nb_win_trades'])
        df['average_loss'] = np.where(df['nb_loss_trades'] == 0, 0.0,
                                      df['loss_sum'] / df['nb_loss_trades'])

        # Win rate = number of profitable trades / number of trades
        df['winrate'] = df['nb_win_trades'] / df['nb_trades']

        # risk_reward_ratio = average win / average loss
        df['risk_reward_ratio'] = df['average_win'] / df['average_loss']

        # required_risk_reward = (1 / winrate) - 1
        df['required_risk_reward'] = (1 / df['winrate']) - 1

        # expectancy = (risk_reward_ratio * winrate) - (lossrate)
        df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate'])

        # sort by expectancy and stoploss
        df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby(
            'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index()

        final = {}
        for x in df.itertuples():
            final[x.pair] = PairInfo(
                x.stoploss,
                x.winrate,
                x.risk_reward_ratio,
                x.required_risk_reward,
                x.expectancy,
                x.nb_trades,
                x.avg_trade_duration
            )

        # Returning a list of pairs in order of "expectancy"
        return final

    def _find_trades_for_stoploss_range(self, df, pair, stoploss_range):
        buy_column = df['buy'].values
        sell_column = df['sell'].values
        date_column = df['date'].values
        ohlc_columns = df[['open', 'high', 'low', 'close']].values

        result: list = []
        for stoploss in stoploss_range:
            result += self._detect_next_stop_or_sell_point(
                buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair
            )

        return result

    def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,
                                        ohlc_columns, stoploss, pair):
        """
        Iterate through ohlc_columns in order to find the next trade
        Next trade opens from the first buy signal noticed to
        The sell or stoploss signal after it.
        It then cuts OHLC, buy_column, sell_column and date_column.
        Cut from (the exit trade index) + 1.

        Author: https://github.com/mishaker
        """

        result: list = []
        start_point = 0

        while True:
            open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal)

            # Return empty if we don't find trade entry (i.e. buy==1) or
            # we find a buy but at the end of array
            if open_trade_index == -1 or open_trade_index == len(buy_column) - 1:
                break
            else:
                # When a buy signal is seen,
                # trade opens in reality on the next candle
                open_trade_index += 1

            open_price = ohlc_columns[open_trade_index, 0]
            stop_price = (open_price * (stoploss + 1))

            # Searching for the index where stoploss is hit
            stop_index = utf1st.find_1st(
                ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller)

            # If we don't find it then we assume stop_index will be far in future (infinite number)
            if stop_index == -1:
                stop_index = float('inf')

            # Searching for the index where sell is hit
            sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal)

            # If we don't find it then we assume sell_index will be far in future (infinite number)
            if sell_index == -1:
                sell_index = float('inf')

            # Check if we don't find any stop or sell point (in that case trade remains open)
            # It is not interesting for Edge to consider it so we simply ignore the trade
            # And stop iterating there is no more entry
            if stop_index == sell_index == float('inf'):
                break

            if stop_index <= sell_index:
                exit_index = open_trade_index + stop_index
                exit_type = SellType.STOP_LOSS
                exit_price = stop_price
            elif stop_index > sell_index:
                # If exit is SELL then we exit at the next candle
                exit_index = open_trade_index + sell_index + 1

                # Check if we have the next candle
                if len(ohlc_columns) - 1 < exit_index:
                    break

                exit_type = SellType.SELL_SIGNAL
                exit_price = ohlc_columns[exit_index, 0]

            trade = {'pair': pair,
                     'stoploss': stoploss,
                     'profit_ratio': '',
                     'profit_abs': '',
                     'open_date': date_column[open_trade_index],
                     'close_date': date_column[exit_index],
                     'trade_duration': '',
                     'open_rate': round(open_price, 15),
                     'close_rate': round(exit_price, 15),
                     'exit_type': exit_type
                     }

            result.append(trade)

            # Giving a view of exit_index till the end of array
            buy_column = buy_column[exit_index:]
            sell_column = sell_column[exit_index:]
            date_column = date_column[exit_index:]
            ohlc_columns = ohlc_columns[exit_index:]
            start_point += exit_index

        return result

1	# pragma pylint: disable=W0603
2	""" Edge positioning package """	1✔
3	import logging	1✔
4	from collections import defaultdict	1✔
5	from copy import deepcopy	1✔
6	from typing import Any, Dict, List, NamedTuple	1✔
7
8	import arrow	1✔
9	import numpy as np	1✔
10	import utils_find_1st as utf1st	1✔
11	from pandas import DataFrame	1✔
12
13	from freqtrade.configuration import TimeRange	1✔
14	from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT	1✔
15	from freqtrade.data.history import get_timerange, load_data, refresh_data	1✔
16	from freqtrade.enums import RunMode, SellType	1✔
17	from freqtrade.exceptions import OperationalException	1✔
18	from freqtrade.exchange.exchange import timeframe_to_seconds	1✔
19	from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist	1✔
20	from freqtrade.strategy.interface import IStrategy	1✔
21
22
23	logger = logging.getLogger(__name__)	1✔
24
25
26	class PairInfo(NamedTuple):	1✔
27	stoploss: float	1✔
28	winrate: float	1✔
29	risk_reward_ratio: float	1✔
30	required_risk_reward: float	1✔
31	expectancy: float	1✔
32	nb_trades: int	1✔
33	avg_trade_duration: float	1✔
34
35
36	class Edge:	1✔
37	"""
38	Calculates Win Rate, Risk Reward Ratio, Expectancy
39	against historical data for a give set of markets and a strategy
40	it then adjusts stoploss and position size accordingly
41	and force it into the strategy
42	Author: https://github.com/mishaker
43	"""
44
45	config: Dict = {}	1✔
46	_cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs	1✔
47
48	def __init__(self, config: Dict[str, Any], exchange, strategy) -> None:	1✔
49
50	self.config = config	1✔
51	self.exchange = exchange	1✔
52	self.strategy: IStrategy = strategy	1✔
53
54	self.edge_config = self.config.get('edge', {})	1✔
55	self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs	1✔
56	self._final_pairs: list = []	1✔
57
58	# checking max_open_trades. it should be -1 as with Edge
59	# the number of trades is determined by position size
60	if self.config['max_open_trades'] != float('inf'):	1✔
61	logger.critical('max_open_trades should be -1 in config !')	1✔
62
63	if self.config['stake_amount'] != UNLIMITED_STAKE_AMOUNT:	1✔
64	raise OperationalException('Edge works only with unlimited stake amount')	1✔
65
66	self._capital_ratio: float = self.config['tradable_balance_ratio']	1✔
67	self._allowed_risk: float = self.edge_config.get('allowed_risk')	1✔
68	self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14)	1✔
69	self._last_updated: int = 0 # Timestamp of pairs last updated time	1✔
70	self._refresh_pairs = True	1✔
71
72	self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01))	1✔
73	self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05))	1✔
74	self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001))	1✔
75
76	# calculating stoploss range
77	self._stoploss_range = np.arange(	1✔
78	self._stoploss_range_min,
79	self._stoploss_range_max,
80	self._stoploss_range_step
81	)
82
83	self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift(	1✔
84	days=-1 * self._since_number_of_days).format('YYYYMMDD'))
85	if config.get('fee'):	1✔
86	self.fee = config['fee']	1✔
87	else:
88	try:	1✔
89	self.fee = self.exchange.get_fee(symbol=expand_pairlist(	1✔
90	self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
91	except IndexError:	1✔
92	self.fee = None	1✔
93
94	def calculate(self, pairs: List[str]) -> bool:	1✔
95	if self.fee is None and pairs:	1✔
96	self.fee = self.exchange.get_fee(pairs[0])	1✔
97
98	heartbeat = self.edge_config.get('process_throttle_secs')	1✔
99
100	if (self._last_updated > 0) and (	1✔
101	self._last_updated + heartbeat > arrow.utcnow().int_timestamp):
102	return False	1✔
103
104	data: Dict[str, Any] = {}	1✔
105	logger.info('Using stake_currency: %s ...', self.config['stake_currency'])	1✔
106	logger.info('Using local backtesting data (using whitelist in given config) ...')	1✔
107
108	if self._refresh_pairs:	1✔
109	timerange_startup = deepcopy(self._timerange)	1✔
110	timerange_startup.subtract_start(timeframe_to_seconds(	1✔
111	self.strategy.timeframe) * self.strategy.startup_candle_count)
112	refresh_data(	1✔
113	datadir=self.config['datadir'],
114	pairs=pairs,
115	exchange=self.exchange,
116	timeframe=self.strategy.timeframe,
117	timerange=timerange_startup,
118	data_format=self.config.get('dataformat_ohlcv', 'json'),
119	)
120	# Download informative pairs too
121	res = defaultdict(list)	1✔
122	for p, t in self.strategy.gather_informative_pairs():	1✔
123	res[t].append(p)	×
124	for timeframe, inf_pairs in res.items():	1✔
125	timerange_startup = deepcopy(self._timerange)	×
126	timerange_startup.subtract_start(timeframe_to_seconds(	×
127	timeframe) * self.strategy.startup_candle_count)
128	refresh_data(	×
129	datadir=self.config['datadir'],
130	pairs=inf_pairs,
131	exchange=self.exchange,
132	timeframe=timeframe,
133	timerange=timerange_startup,
134	data_format=self.config.get('dataformat_ohlcv', 'json'),
135	)
136
137	data = load_data(	1✔
138	datadir=self.config['datadir'],
139	pairs=pairs,
140	timeframe=self.strategy.timeframe,
141	timerange=self._timerange,
142	startup_candles=self.strategy.startup_candle_count,
143	data_format=self.config.get('dataformat_ohlcv', 'json'),
144	)
145
146	if not data:	1✔
147	# Reinitializing cached pairs
148	self._cached_pairs = {}	1✔
149	logger.critical("No data found. Edge is stopped ...")	1✔
150	return False	1✔
151	# Fake run-mode to Edge
152	prior_rm = self.config['runmode']	1✔
153	self.config['runmode'] = RunMode.EDGE	1✔
154	preprocessed = self.strategy.advise_all_indicators(data)	1✔
155	self.config['runmode'] = prior_rm	1✔
156
157	# Print timeframe
158	min_date, max_date = get_timerange(preprocessed)	1✔
159	logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '	1✔
160	f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
161	f'({(max_date - min_date).days} days)..')
162	headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low']	1✔
163
164	trades: list = []	1✔
165	for pair, pair_data in preprocessed.items():	1✔
166	# Sorting dataframe by date and reset index
167	pair_data = pair_data.sort_values(by=['date'])	1✔
168	pair_data = pair_data.reset_index(drop=True)	1✔
169
170	df_analyzed = self.strategy.advise_sell(	1✔
171	self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
172
173	trades += self._find_trades_for_stoploss_range(df_analyzed, pair, self._stoploss_range)	1✔
174
175	# If no trade found then exit
176	if len(trades) == 0:	1✔
177	logger.info("No trades found.")	1✔
178	return False	1✔
179
180	# Fill missing, calculable columns, profit, duration , abs etc.
181	trades_df = self._fill_calculable_fields(DataFrame(trades))	1✔
182	self._cached_pairs = self._process_expectancy(trades_df)	1✔
183	self._last_updated = arrow.utcnow().int_timestamp	1✔
184
185	return True	1✔
186
187	def stake_amount(self, pair: str, free_capital: float,	1✔
188	total_capital: float, capital_in_trade: float) -> float:
189	stoploss = self.stoploss(pair)	1✔
190	available_capital = (total_capital + capital_in_trade) * self._capital_ratio	1✔
191	allowed_capital_at_risk = available_capital * self._allowed_risk	1✔
192	max_position_size = abs(allowed_capital_at_risk / stoploss)	1✔
193	# Position size must be below available capital.
194	position_size = min(min(max_position_size, free_capital), available_capital)	1✔
195	if pair in self._cached_pairs:	1✔
196	logger.info(	1✔
197	'winrate: %s, expectancy: %s, position size: %s, pair: %s,'
198	' capital in trade: %s, free capital: %s, total capital: %s,'
199	' stoploss: %s, available capital: %s.',
200	self._cached_pairs[pair].winrate,
201	self._cached_pairs[pair].expectancy,
202	position_size, pair,
203	capital_in_trade, free_capital, total_capital,
204	stoploss, available_capital
205	)
206	return round(position_size, 15)	1✔
207
208	def stoploss(self, pair: str) -> float:	1✔
209	if pair in self._cached_pairs:	1✔
210	return self._cached_pairs[pair].stoploss	1✔
211	else:
212	logger.warning(f'Tried to access stoploss of non-existing pair {pair}, '	1✔
213	'strategy stoploss is returned instead.')
214	return self.strategy.stoploss	1✔
215
216	def adjust(self, pairs: List[str]) -> list:	1✔
217	"""
218	Filters out and sorts "pairs" according to Edge calculated pairs
219	"""
220	final = []	1✔
221	for pair, info in self._cached_pairs.items():	1✔
222	if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \	1✔
223	info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \
224	pair in pairs:
225	final.append(pair)	1✔
226
227	if self._final_pairs != final:	1✔
228	self._final_pairs = final	1✔
229	if self._final_pairs:	1✔
230	logger.info(	1✔
231	'Minimum expectancy and minimum winrate are met only for %s,'
232	' so other pairs are filtered out.',
233	self._final_pairs
234	)
235	else:
236	logger.info(	×
237	'Edge removed all pairs as no pair with minimum expectancy '
238	'and minimum winrate was found !'
239	)
240
241	return self._final_pairs	1✔
242
243	def accepted_pairs(self) -> List[Dict[str, Any]]:	1✔
244	"""
245	return a list of accepted pairs along with their winrate, expectancy and stoploss
246	"""
247	final = []	1✔
248	for pair, info in self._cached_pairs.items():	1✔
249	if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \	1✔
250	info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)):
251	final.append({	1✔
252	'Pair': pair,
253	'Winrate': info.winrate,
254	'Expectancy': info.expectancy,
255	'Stoploss': info.stoploss,
256	})
257	return final	1✔
258
259	def _fill_calculable_fields(self, result: DataFrame) -> DataFrame:	1✔
260	"""
261	The result frame contains a number of columns that are calculable
262	from other columns. These are left blank till all rows are added,
263	to be populated in single vector calls.
264
265	Columns to be populated are:
266	- Profit
267	- trade duration
268	- profit abs
269	:param result Dataframe
270	:return: result Dataframe
271	"""
272	# We set stake amount to an arbitrary amount, as it doesn't change the calculation.
273	# All returned values are relative, they are defined as ratios.
274	stake = 0.015	1✔
275
276	result['trade_duration'] = result['close_date'] - result['open_date']	1✔
277
278	result['trade_duration'] = result['trade_duration'].map(	1✔
279	lambda x: int(x.total_seconds() / 60))
280
281	# Spends, Takes, Profit, Absolute Profit
282
283	# Buy Price
284	result['buy_vol'] = stake / result['open_rate'] # How many target are we buying	1✔
285	result['buy_fee'] = stake * self.fee	1✔
286	result['buy_spend'] = stake + result['buy_fee'] # How much we're spending	1✔
287
288	# Sell price
289	result['sell_sum'] = result['buy_vol'] * result['close_rate']	1✔
290	result['sell_fee'] = result['sell_sum'] * self.fee	1✔
291	result['sell_take'] = result['sell_sum'] - result['sell_fee']	1✔
292
293	# profit_ratio
294	result['profit_ratio'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend']	1✔
295
296	# Absolute profit
297	result['profit_abs'] = result['sell_take'] - result['buy_spend']	1✔
298
299	return result	1✔
300
301	def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]:	1✔
302	"""
303	This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs
304	The calculation will be done per pair and per strategy.
305	"""
306	# Removing pairs having less than min_trades_number
307	min_trades_number = self.edge_config.get('min_trade_number', 10)	1✔
308	results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number)	1✔
309	###################################
310
311	# Removing outliers (Only Pumps) from the dataset
312	# The method to detect outliers is to calculate standard deviation
313	# Then every value more than (standard deviation + 2*average) is out (pump)
314	#
315	# Removing Pumps
316	if self.edge_config.get('remove_pumps', False):	1✔
317	results = results[results['profit_abs'] < 2 * results['profit_abs'].std()	1✔
318	+ results['profit_abs'].mean()]
319	##########################################################################
320
321	# Removing trades having a duration more than X minutes (set in config)
322	max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440)	1✔
323	results = results[results.trade_duration < max_trade_duration]	1✔
324	#######################################################################
325
326	if results.empty:	1✔
327	return {}	1✔
328
329	groupby_aggregator = {	1✔
330	'profit_abs': [
331	('nb_trades', 'count'), # number of all trades
332	('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades
333	('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades
334	('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades
335	],
336	'trade_duration': [('avg_trade_duration', 'mean')]
337	}
338
339	# Group by (pair and stoploss) by applying above aggregator
340	df = results.groupby(['pair', 'stoploss'])[['profit_abs', 'trade_duration']].agg(	1✔
341	groupby_aggregator).reset_index(col_level=1)
342
343	# Dropping level 0 as we don't need it
344	df.columns = df.columns.droplevel(0)	1✔
345
346	# Calculating number of losing trades, average win and average loss
347	df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades']	1✔
348	df['average_win'] = np.where(df['nb_win_trades'] == 0, 0.0,	1✔
349	df['profit_sum'] / df['nb_win_trades'])
350	df['average_loss'] = np.where(df['nb_loss_trades'] == 0, 0.0,	1✔
351	df['loss_sum'] / df['nb_loss_trades'])
352
353	# Win rate = number of profitable trades / number of trades
354	df['winrate'] = df['nb_win_trades'] / df['nb_trades']	1✔
355
356	# risk_reward_ratio = average win / average loss
357	df['risk_reward_ratio'] = df['average_win'] / df['average_loss']	1✔
358
359	# required_risk_reward = (1 / winrate) - 1
360	df['required_risk_reward'] = (1 / df['winrate']) - 1	1✔
361
362	# expectancy = (risk_reward_ratio * winrate) - (lossrate)
363	df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate'])	1✔
364
365	# sort by expectancy and stoploss
366	df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby(	1✔
367	'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index()
368
369	final = {}	1✔
370	for x in df.itertuples():	1✔
371	final[x.pair] = PairInfo(	1✔
372	x.stoploss,
373	x.winrate,
374	x.risk_reward_ratio,
375	x.required_risk_reward,
376	x.expectancy,
377	x.nb_trades,
378	x.avg_trade_duration
379	)
380
381	# Returning a list of pairs in order of "expectancy"
382	return final	1✔
383
384	def _find_trades_for_stoploss_range(self, df, pair, stoploss_range):	1✔
385	buy_column = df['buy'].values	1✔
386	sell_column = df['sell'].values	1✔
387	date_column = df['date'].values	1✔
388	ohlc_columns = df[['open', 'high', 'low', 'close']].values	1✔
389
390	result: list = []	1✔
391	for stoploss in stoploss_range:	1✔
392	result += self._detect_next_stop_or_sell_point(	1✔
393	buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair
394	)
395
396	return result	1✔
397
398	def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column,	1✔
399	ohlc_columns, stoploss, pair):
400	"""
401	Iterate through ohlc_columns in order to find the next trade
402	Next trade opens from the first buy signal noticed to
403	The sell or stoploss signal after it.
404	It then cuts OHLC, buy_column, sell_column and date_column.
405	Cut from (the exit trade index) + 1.
406
407	Author: https://github.com/mishaker
408	"""
409
410	result: list = []	1✔
411	start_point = 0	1✔
412
413	while True:
414	open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal)	1✔
415
416	# Return empty if we don't find trade entry (i.e. buy==1) or
417	# we find a buy but at the end of array
418	if open_trade_index == -1 or open_trade_index == len(buy_column) - 1:	1✔
419	break	1✔
420	else:
421	# When a buy signal is seen,
422	# trade opens in reality on the next candle
423	open_trade_index += 1	1✔
424
425	open_price = ohlc_columns[open_trade_index, 0]	1✔
426	stop_price = (open_price * (stoploss + 1))	1✔
427
428	# Searching for the index where stoploss is hit
429	stop_index = utf1st.find_1st(	1✔
430	ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller)
431
432	# If we don't find it then we assume stop_index will be far in future (infinite number)
433	if stop_index == -1:	1✔
434	stop_index = float('inf')	1✔
435
436	# Searching for the index where sell is hit
437	sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal)	1✔
438
439	# If we don't find it then we assume sell_index will be far in future (infinite number)
440	if sell_index == -1:	1✔
441	sell_index = float('inf')	1✔
442
443	# Check if we don't find any stop or sell point (in that case trade remains open)
444	# It is not interesting for Edge to consider it so we simply ignore the trade
445	# And stop iterating there is no more entry
446	if stop_index == sell_index == float('inf'):	1✔
447	break	×
448
449	if stop_index <= sell_index:	1✔
450	exit_index = open_trade_index + stop_index	1✔
451	exit_type = SellType.STOP_LOSS	1✔
452	exit_price = stop_price	1✔
453	elif stop_index > sell_index:	1✔
454	# If exit is SELL then we exit at the next candle
455	exit_index = open_trade_index + sell_index + 1	1✔
456
457	# Check if we have the next candle
458	if len(ohlc_columns) - 1 < exit_index:	1✔
459	break	1✔
460
461	exit_type = SellType.SELL_SIGNAL	1✔
462	exit_price = ohlc_columns[exit_index, 0]	1✔
463
464	trade = {'pair': pair,	1✔
465	'stoploss': stoploss,
466	'profit_ratio': '',
467	'profit_abs': '',
468	'open_date': date_column[open_trade_index],
469	'close_date': date_column[exit_index],
470	'trade_duration': '',
471	'open_rate': round(open_price, 15),
472	'close_rate': round(exit_price, 15),
473	'exit_type': exit_type
474	}
475
476	result.append(trade)	1✔
477
478	# Giving a view of exit_index till the end of array
479	buy_column = buy_column[exit_index:]	1✔
480	sell_column = sell_column[exit_index:]	1✔
481	date_column = date_column[exit_index:]	1✔
482	ohlc_columns = ohlc_columns[exit_index:]	1✔
483	start_point += exit_index	1✔
484
485	return result	1✔

freqtrade / freqtrade / 1557291546

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous