Merge branch 'freqtrade:develop' into freqtrade-develop

stash86 · Jan 16, 2024 · 03bd6a5 · 03bd6a5
2 parents 5084722 + c248bb2
commit 03bd6a5
Show file tree

Hide file tree

Showing 37 changed files with 252 additions and 133 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-20.04, ubuntu-22.04 ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -125,7 +125,7 @@ jobs:
     strategy:
       matrix:
         os: [ "macos-latest", "macos-13" ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4
@@ -238,7 +238,7 @@ jobs:
     strategy:
       matrix:
         os: [ windows-latest ]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - uses: actions/checkout@v4

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -31,7 +31,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.1.11'
+    rev: 'v0.1.13'
     hooks:
       - id: ruff
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.6-slim-bookworm as base
+FROM python:3.11.7-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf
@@ -1,4 +1,4 @@
-FROM python:3.11.6-slim-bookworm as base
+FROM python:3.11.7-slim-bookworm as base
 
 # Setup env
 ENV LANG C.UTF-8

diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter
@@ -1,8 +1,8 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-# Pin jupyter-client to avoid tornado version conflict
-RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir
+# Pin prompt-toolkit to avoid questionary version conflict
+RUN pip install jupyterlab "prompt-toolkit<=3.0.36" jupyter-client --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []
diff --git a/docker/docker-compose-jupyter.yml b/docker/docker-compose-jupyter.yml
@@ -6,7 +6,7 @@ services:
       context: ..
       dockerfile: docker/Dockerfile.jupyter
     restart: unless-stopped
-    container_name: freqtrade
+    # container_name: freqtrade
     ports:
       - "127.0.0.1:8888:8888"
     volumes:

diff --git a/docs/hyperopt.md b/docs/hyperopt.md
@@ -439,7 +439,7 @@ While this strategy is most likely too simple to provide consistent profit, it s
 ??? Hint "Performance tip"
     During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage
 
-    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality.
+    * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` will be calculated every epoch, you don't need to use `.range` functionality.
     * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value.
 
     These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues.
@@ -926,6 +926,12 @@ Once the optimized strategy has been implemented into your strategy, you should
 
 To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
 
-Should results not match, please double-check to make sure you transferred all conditions correctly.
-Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy.
-You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+### Why do my backtest results not match my hyperopt results?
+Should results not match, check the following factors:
+
+* You may have added parameters to hyperopt in `populate_indicators()` where they will be calculated only once **for all epochs**. If you are, for example, trying to optimise multiple SMA timeperiod values, the hyperoptable timeperiod parameter should be placed in `populate_entry_trend()` which is calculated every epoch. See [Optimizing an indicator parameter](https://www.freqtrade.io/en/stable/hyperopt/#optimizing-an-indicator-parameter).
+* If you have disabled the auto-export of hyperopt parameters into the JSON parameters file, double-check to make sure you transferred all hyperopted values into your strategy correctly.
+* Check the logs to verify what parameters are being set and what values are being used.
+* Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. Check the logs of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`).
+* Verify that you do not have an unexpected parameters JSON file overriding the parameters or the default hyperopt settings in your strategy.
+* Verify that any protections that are enabled in backtesting are also enabled when hyperopting, and vice versa. When using `--space protection`, protections are auto-enabled for hyperopting.
diff --git a/docs/includes/showcase.md b/docs/includes/showcase.md
@@ -5,7 +5,7 @@ This section will highlight a few projects from members of the community.
 - [Example freqtrade strategies](https://github.com/freqtrade/freqtrade-strategies/)
 - [FrequentHippo - Grafana dashboard with dry/live runs and backtests](http://frequenthippo.ddns.net:3000/) (by hippocritical).
 - [Online pairlist generator](https://remotepairlist.com/) (by Blood4rc).
-- [Freqtrade Backtesting Project](https://bt.robot.co.network/) (by Blood4rc).
+- [Freqtrade Backtesting Project](https://strat.ninja/) (by Blood4rc).
 - [Freqtrade analysis notebook](https://github.com/froggleston/freqtrade_analysis_notebook) (by Froggleston).
 - [TUI for freqtrade](https://github.com/froggleston/freqtrade-frogtrade9000) (by Froggleston).
 - [Bot Academy](https://botacademy.ddns.net/) (by stash86) - Blog about crypto bot projects.
diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt
@@ -1,6 +1,6 @@
-markdown==3.5.1
+markdown==3.5.2
 mkdocs==1.5.3
 mkdocs-material==9.5.3
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.7
-jinja2==3.1.2
+jinja2==3.1.3
diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py
@@ -219,27 +219,35 @@ def _build_subcommands(self) -> None:
                                                 )
 
         # Add trade subcommand
-        trade_cmd = subparsers.add_parser('trade', help='Trade module.',
-                                          parents=[_common_parser, _strategy_parser])
+        trade_cmd = subparsers.add_parser(
+            'trade',
+            help='Trade module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         trade_cmd.set_defaults(func=start_trading)
         self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd)
 
         # add create-userdir subcommand
-        create_userdir_cmd = subparsers.add_parser('create-userdir',
-                                                   help="Create user-data directory.",
-                                                   )
+        create_userdir_cmd = subparsers.add_parser(
+            'create-userdir',
+            help="Create user-data directory.",
+        )
         create_userdir_cmd.set_defaults(func=start_create_userdir)
         self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd)
 
         # add new-config subcommand
-        build_config_cmd = subparsers.add_parser('new-config',
-                                                 help="Create new config")
+        build_config_cmd = subparsers.add_parser(
+            'new-config',
+            help="Create new config",
+        )
         build_config_cmd.set_defaults(func=start_new_config)
         self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd)
 
         # add new-strategy subcommand
-        build_strategy_cmd = subparsers.add_parser('new-strategy',
-                                                   help="Create new strategy")
+        build_strategy_cmd = subparsers.add_parser(
+            'new-strategy',
+            help="Create new strategy",
+        )
         build_strategy_cmd.set_defaults(func=start_new_strategy)
         self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd)
 
@@ -289,8 +297,11 @@ def _build_subcommands(self) -> None:
         self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd)
 
         # Add backtesting subcommand
-        backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.',
-                                                parents=[_common_parser, _strategy_parser])
+        backtesting_cmd = subparsers.add_parser(
+            'backtesting',
+            help='Backtesting module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         backtesting_cmd.set_defaults(func=start_backtesting)
         self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd)
 
@@ -304,22 +315,29 @@ def _build_subcommands(self) -> None:
         self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd)
 
         # Add backtesting analysis subcommand
-        analysis_cmd = subparsers.add_parser('backtesting-analysis',
-                                             help='Backtest Analysis module.',
-                                             parents=[_common_parser])
+        analysis_cmd = subparsers.add_parser(
+            'backtesting-analysis',
+            help='Backtest Analysis module.',
+            parents=[_common_parser]
+        )
         analysis_cmd.set_defaults(func=start_analysis_entries_exits)
         self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd)
 
         # Add edge subcommand
-        edge_cmd = subparsers.add_parser('edge', help='Edge module.',
-                                         parents=[_common_parser, _strategy_parser])
+        edge_cmd = subparsers.add_parser(
+            'edge',
+            help='Edge module.',
+            parents=[_common_parser, _strategy_parser]
+        )
         edge_cmd.set_defaults(func=start_edge)
         self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd)
 
         # Add hyperopt subcommand
-        hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.',
-                                             parents=[_common_parser, _strategy_parser],
-                                             )
+        hyperopt_cmd = subparsers.add_parser(
+            'hyperopt',
+            help='Hyperopt module.',
+            parents=[_common_parser, _strategy_parser],
+        )
         hyperopt_cmd.set_defaults(func=start_hyperopt)
         self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd)
 
@@ -447,25 +465,29 @@ def _build_subcommands(self) -> None:
         self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd)
 
         # Add webserver subcommand
-        webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.',
-                                              parents=[_common_parser])
+        webserver_cmd = subparsers.add_parser(
+            'webserver',
+            help='Webserver module.',
+            parents=[_common_parser]
+        )
         webserver_cmd.set_defaults(func=start_webserver)
         self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd)
 
         # Add strategy_updater subcommand
-        strategy_updater_cmd = subparsers.add_parser('strategy-updater',
-                                                     help='updates outdated strategy'
-                                                          'files to the current version',
-                                                     parents=[_common_parser])
+        strategy_updater_cmd = subparsers.add_parser(
+            'strategy-updater',
+            help='updates outdated strategy files to the current version',
+            parents=[_common_parser]
+        )
         strategy_updater_cmd.set_defaults(func=start_strategy_update)
         self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd)
 
         # Add lookahead_analysis subcommand
         lookahead_analayis_cmd = subparsers.add_parser(
             'lookahead-analysis',
             help="Check for potential look ahead bias.",
-            parents=[_common_parser, _strategy_parser])
-
+            parents=[_common_parser, _strategy_parser]
+        )
         lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis)
 
         self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS,
@@ -475,8 +497,8 @@ def _build_subcommands(self) -> None:
         recursive_analayis_cmd = subparsers.add_parser(
             'recursive-analysis',
             help="Check for potential recursive formula issue.",
-            parents=[_common_parser, _strategy_parser])
-
+            parents=[_common_parser, _strategy_parser]
+        )
         recursive_analayis_cmd.set_defaults(func=start_recursive_analysis)
 
         self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS,

diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py
@@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     """
     Test Pairlist configuration
     """
+    from freqtrade.persistence import FtNoDBContext
     from freqtrade.plugins.pairlistmanager import PairListManager
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
 
@@ -24,11 +25,12 @@ def start_test_pairlist(args: Dict[str, Any]) -> None:
     if not quote_currencies:
         quote_currencies = [config.get('stake_currency')]
     results = {}
-    for curr in quote_currencies:
-        config['stake_currency'] = curr
-        pairlists = PairListManager(exchange, config)
-        pairlists.refresh_pairlist()
-        results[curr] = pairlists.whitelist
+    with FtNoDBContext():
+        for curr in quote_currencies:
+            config['stake_currency'] = curr
+            pairlists = PairListManager(exchange, config)
+            pairlists.refresh_pairlist()
+            results[curr] = pairlists.whitelist
 
     for curr, pairlist in results.items():
         if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False):

diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py
@@ -68,8 +68,10 @@ def load_config(self) -> Dict[str, Any]:
         config: Config = load_from_files(self.args.get("config", []))
 
         # Load environment variables
-        env_data = enironment_vars_to_dict()
-        config = deep_merge_dicts(env_data, config)
+        from freqtrade.commands.arguments import NO_CONF_ALLOWED
+        if self.args.get('command') not in NO_CONF_ALLOWED:
+            env_data = enironment_vars_to_dict()
+            config = deep_merge_dicts(env_data, config)
 
         # Normalize config
         if 'internals' not in config:

diff --git a/freqtrade/configuration/environment_vars.py b/freqtrade/configuration/environment_vars.py
@@ -9,7 +9,7 @@
 logger = logging.getLogger(__name__)
 
 
-def get_var_typed(val):
+def _get_var_typed(val):
     try:
         return int(val)
     except ValueError:
@@ -24,7 +24,7 @@ def get_var_typed(val):
     return val
 
 
-def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
+def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]:
     """
     Environment variables must be prefixed with FREQTRADE.
     FREQTRADE__{section}__{key}
@@ -40,7 +40,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str,
             logger.info(f"Loading variable '{env_var}'")
             key = env_var.replace(prefix, '')
             for k in reversed(key.split('__')):
-                val = {k.lower(): get_var_typed(val)
+                val = {k.lower(): _get_var_typed(val)
                        if not isinstance(val, dict) and k not in no_convert else val}
             relevant_vars = deep_merge_dicts(val, relevant_vars)
     return relevant_vars
@@ -52,4 +52,4 @@ def enironment_vars_to_dict() -> Dict[str, Any]:
     Relevant variables must follow the FREQTRADE__{section}__{key} pattern
     :return: Nested dict based on available and relevant variables.
     """
-    return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)
+    return _flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX)
diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py
@@ -1453,19 +1453,19 @@ def replace_order(self, order: Dict, order_obj: Optional[Order], trade: Trade) -
             # New candle
             proposed_rate = self.exchange.get_rate(
                 trade.pair, side='entry', is_short=trade.is_short, refresh=True)
-            adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price,
-                                                         default_retval=order_obj.price)(
+            adjusted_entry_price = strategy_safe_wrapper(
+                self.strategy.adjust_entry_price, default_retval=order_obj.safe_placement_price)(
                 trade=trade, order=order_obj, pair=trade.pair,
                 current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate,
-                current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag,
+                current_order_rate=order_obj.safe_placement_price, entry_tag=trade.enter_tag,
                 side=trade.trade_direction)
 
             replacing = True
             cancel_reason = constants.CANCEL_REASON['REPLACE']
             if not adjusted_entry_price:
                 replacing = False
                 cancel_reason = constants.CANCEL_REASON['USER_CANCEL']
-            if order_obj.price != adjusted_entry_price:
+            if order_obj.safe_placement_price != adjusted_entry_price:
                 # cancel existing order if new price is supplied or None
                 res = self.handle_cancel_enter(trade, order, order_obj, cancel_reason,
                                                replacing=replacing)

diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py
@@ -33,7 +33,8 @@
                                                  show_backtest_results,
                                                  store_backtest_analysis_results,
                                                  store_backtest_stats)
-from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade
+from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use,
+                                   enable_database_use)
 from freqtrade.plugins.pairlistmanager import PairListManager
 from freqtrade.plugins.protectionmanager import ProtectionManager
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
@@ -177,8 +178,7 @@ def _validate_pairlists_for_backtesting(self):
     @staticmethod
     def cleanup():
         LoggingMixin.show_output = True
-        PairLocks.use_db = True
-        Trade.use_db = True
+        enable_database_use()
 
     def init_backtest_detail(self) -> None:
         # Load detail timeframe if specified
@@ -325,9 +325,7 @@ def load_bt_data_detail(self) -> None:
             self.futures_data = {}
 
     def disable_database_use(self):
-        PairLocks.use_db = False
-        PairLocks.timeframe = self.timeframe
-        Trade.use_db = False
+        disable_database_use(self.timeframe)
 
     def prepare_backtest(self, enable_protections):
         """

diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py
@@ -54,7 +54,7 @@ def fill_full_varholder(self):
             self.full_varHolder.from_dt = parsed_timerange.startdt
 
         if parsed_timerange.stopdt is None:
-            self.full_varHolder.to_dt = datetime.utcnow()
+            self.full_varHolder.to_dt = datetime.now(timezone.utc)
         else:
             self.full_varHolder.to_dt = parsed_timerange.stopdt
 

diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py
@@ -4,3 +4,5 @@
 from freqtrade.persistence.models import init_db
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.persistence.trade_model import LocalTrade, Order, Trade
+from freqtrade.persistence.usedb_context import (FtNoDBContext, disable_database_use,
+                                                 enable_database_use)