diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d1df9a0517..3e3ee97942e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04, ubuntu-22.04 ] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -125,7 +125,7 @@ jobs: strategy: matrix: os: [ "macos-latest", "macos-13" ] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 @@ -238,7 +238,7 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1c630f3efd..dca08dfb8f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: 'v0.1.11' + rev: 'v0.1.13' hooks: - id: ruff diff --git a/Dockerfile b/Dockerfile index 38f9ca788bc..e5a33df87af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.6-slim-bookworm as base +FROM python:3.11.7-slim-bookworm as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index c8efa42328c..4cb8f5fea89 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.11.6-slim-bookworm as base +FROM python:3.11.7-slim-bookworm as base # Setup env ENV LANG C.UTF-8 diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter index d86980bdf7a..79fefb3a79a 100644 --- a/docker/Dockerfile.jupyter +++ b/docker/Dockerfile.jupyter @@ -1,8 +1,8 @@ FROM freqtradeorg/freqtrade:develop_plot -# Pin jupyter-client to avoid tornado version conflict -RUN pip install jupyterlab jupyter-client==7.3.4 --user --no-cache-dir +# Pin prompt-toolkit to avoid questionary version conflict +RUN pip install jupyterlab "prompt-toolkit<=3.0.36" jupyter-client --user --no-cache-dir # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/docker/docker-compose-jupyter.yml b/docker/docker-compose-jupyter.yml index 3df82365f00..b72c8df84e9 100644 --- a/docker/docker-compose-jupyter.yml +++ b/docker/docker-compose-jupyter.yml @@ -6,7 +6,7 @@ services: context: .. dockerfile: docker/Dockerfile.jupyter restart: unless-stopped - container_name: freqtrade + # container_name: freqtrade ports: - "127.0.0.1:8888:8888" volumes: diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 44427919587..d3371d77109 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -439,7 +439,7 @@ While this strategy is most likely too simple to provide consistent profit, it s ??? Hint "Performance tip" During normal hyperopting, indicators are calculated once and supplied to each epoch, linearly increasing RAM usage as a factor of increasing cores. As this also has performance implications, there are two alternatives to reduce RAM usage - * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` gonna be calculated every epochs, you don't need to use `.range` functionality. + * Move `ema_short` and `ema_long` calculations from `populate_indicators()` to `populate_entry_trend()`. Since `populate_entry_trend()` will be calculated every epoch, you don't need to use `.range` functionality. * hyperopt provides `--analyze-per-epoch` which will move the execution of `populate_indicators()` to the epoch process, calculating a single value per parameter per epoch instead of using the `.range` functionality. In this case, `.range` functionality will only return the actually used value. These alternatives will reduce RAM usage, but increase CPU usage. However, your hyperopting run will be less likely to fail due to Out Of Memory (OOM) issues. @@ -926,6 +926,12 @@ Once the optimized strategy has been implemented into your strategy, you should To achieve same the results (number of trades, their durations, profit, etc.) as during Hyperopt, please use the same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. -Should results not match, please double-check to make sure you transferred all conditions correctly. -Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. -You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`). +### Why do my backtest results not match my hyperopt results? +Should results not match, check the following factors: + +* You may have added parameters to hyperopt in `populate_indicators()` where they will be calculated only once **for all epochs**. If you are, for example, trying to optimise multiple SMA timeperiod values, the hyperoptable timeperiod parameter should be placed in `populate_entry_trend()` which is calculated every epoch. See [Optimizing an indicator parameter](https://www.freqtrade.io/en/stable/hyperopt/#optimizing-an-indicator-parameter). +* If you have disabled the auto-export of hyperopt parameters into the JSON parameters file, double-check to make sure you transferred all hyperopted values into your strategy correctly. +* Check the logs to verify what parameters are being set and what values are being used. +* Pay special care to the stoploss, max_open_trades and trailing stoploss parameters, as these are often set in configuration files, which override changes to the strategy. Check the logs of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss`, `max_open_trades` or `trailing_stop`). +* Verify that you do not have an unexpected parameters JSON file overriding the parameters or the default hyperopt settings in your strategy. +* Verify that any protections that are enabled in backtesting are also enabled when hyperopting, and vice versa. When using `--space protection`, protections are auto-enabled for hyperopting. diff --git a/docs/includes/showcase.md b/docs/includes/showcase.md index 297685ad424..766703b9f5e 100644 --- a/docs/includes/showcase.md +++ b/docs/includes/showcase.md @@ -5,7 +5,7 @@ This section will highlight a few projects from members of the community. - [Example freqtrade strategies](https://github.com/freqtrade/freqtrade-strategies/) - [FrequentHippo - Grafana dashboard with dry/live runs and backtests](http://frequenthippo.ddns.net:3000/) (by hippocritical). - [Online pairlist generator](https://remotepairlist.com/) (by Blood4rc). -- [Freqtrade Backtesting Project](https://bt.robot.co.network/) (by Blood4rc). +- [Freqtrade Backtesting Project](https://strat.ninja/) (by Blood4rc). - [Freqtrade analysis notebook](https://github.com/froggleston/freqtrade_analysis_notebook) (by Froggleston). - [TUI for freqtrade](https://github.com/froggleston/freqtrade-frogtrade9000) (by Froggleston). - [Bot Academy](https://botacademy.ddns.net/) (by stash86) - Blog about crypto bot projects. diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index a791492751d..7ec1eb4221c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ -markdown==3.5.1 +markdown==3.5.2 mkdocs==1.5.3 mkdocs-material==9.5.3 mdx_truly_sane_lists==1.3 pymdown-extensions==10.7 -jinja2==3.1.2 +jinja2==3.1.3 diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 26d5824652c..f7216467566 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -219,27 +219,35 @@ def _build_subcommands(self) -> None: ) # Add trade subcommand - trade_cmd = subparsers.add_parser('trade', help='Trade module.', - parents=[_common_parser, _strategy_parser]) + trade_cmd = subparsers.add_parser( + 'trade', + help='Trade module.', + parents=[_common_parser, _strategy_parser] + ) trade_cmd.set_defaults(func=start_trading) self._build_args(optionlist=ARGS_TRADE, parser=trade_cmd) # add create-userdir subcommand - create_userdir_cmd = subparsers.add_parser('create-userdir', - help="Create user-data directory.", - ) + create_userdir_cmd = subparsers.add_parser( + 'create-userdir', + help="Create user-data directory.", + ) create_userdir_cmd.set_defaults(func=start_create_userdir) self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd) # add new-config subcommand - build_config_cmd = subparsers.add_parser('new-config', - help="Create new config") + build_config_cmd = subparsers.add_parser( + 'new-config', + help="Create new config", + ) build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) # add new-strategy subcommand - build_strategy_cmd = subparsers.add_parser('new-strategy', - help="Create new strategy") + build_strategy_cmd = subparsers.add_parser( + 'new-strategy', + help="Create new strategy", + ) build_strategy_cmd.set_defaults(func=start_new_strategy) self._build_args(optionlist=ARGS_BUILD_STRATEGY, parser=build_strategy_cmd) @@ -289,8 +297,11 @@ def _build_subcommands(self) -> None: self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd) # Add backtesting subcommand - backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.', - parents=[_common_parser, _strategy_parser]) + backtesting_cmd = subparsers.add_parser( + 'backtesting', + help='Backtesting module.', + parents=[_common_parser, _strategy_parser] + ) backtesting_cmd.set_defaults(func=start_backtesting) self._build_args(optionlist=ARGS_BACKTEST, parser=backtesting_cmd) @@ -304,22 +315,29 @@ def _build_subcommands(self) -> None: self._build_args(optionlist=ARGS_BACKTEST_SHOW, parser=backtesting_show_cmd) # Add backtesting analysis subcommand - analysis_cmd = subparsers.add_parser('backtesting-analysis', - help='Backtest Analysis module.', - parents=[_common_parser]) + analysis_cmd = subparsers.add_parser( + 'backtesting-analysis', + help='Backtest Analysis module.', + parents=[_common_parser] + ) analysis_cmd.set_defaults(func=start_analysis_entries_exits) self._build_args(optionlist=ARGS_ANALYZE_ENTRIES_EXITS, parser=analysis_cmd) # Add edge subcommand - edge_cmd = subparsers.add_parser('edge', help='Edge module.', - parents=[_common_parser, _strategy_parser]) + edge_cmd = subparsers.add_parser( + 'edge', + help='Edge module.', + parents=[_common_parser, _strategy_parser] + ) edge_cmd.set_defaults(func=start_edge) self._build_args(optionlist=ARGS_EDGE, parser=edge_cmd) # Add hyperopt subcommand - hyperopt_cmd = subparsers.add_parser('hyperopt', help='Hyperopt module.', - parents=[_common_parser, _strategy_parser], - ) + hyperopt_cmd = subparsers.add_parser( + 'hyperopt', + help='Hyperopt module.', + parents=[_common_parser, _strategy_parser], + ) hyperopt_cmd.set_defaults(func=start_hyperopt) self._build_args(optionlist=ARGS_HYPEROPT, parser=hyperopt_cmd) @@ -447,16 +465,20 @@ def _build_subcommands(self) -> None: self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) # Add webserver subcommand - webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.', - parents=[_common_parser]) + webserver_cmd = subparsers.add_parser( + 'webserver', + help='Webserver module.', + parents=[_common_parser] + ) webserver_cmd.set_defaults(func=start_webserver) self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) # Add strategy_updater subcommand - strategy_updater_cmd = subparsers.add_parser('strategy-updater', - help='updates outdated strategy' - 'files to the current version', - parents=[_common_parser]) + strategy_updater_cmd = subparsers.add_parser( + 'strategy-updater', + help='updates outdated strategy files to the current version', + parents=[_common_parser] + ) strategy_updater_cmd.set_defaults(func=start_strategy_update) self._build_args(optionlist=ARGS_STRATEGY_UPDATER, parser=strategy_updater_cmd) @@ -464,8 +486,8 @@ def _build_subcommands(self) -> None: lookahead_analayis_cmd = subparsers.add_parser( 'lookahead-analysis', help="Check for potential look ahead bias.", - parents=[_common_parser, _strategy_parser]) - + parents=[_common_parser, _strategy_parser] + ) lookahead_analayis_cmd.set_defaults(func=start_lookahead_analysis) self._build_args(optionlist=ARGS_LOOKAHEAD_ANALYSIS, @@ -475,8 +497,8 @@ def _build_subcommands(self) -> None: recursive_analayis_cmd = subparsers.add_parser( 'recursive-analysis', help="Check for potential recursive formula issue.", - parents=[_common_parser, _strategy_parser]) - + parents=[_common_parser, _strategy_parser] + ) recursive_analayis_cmd.set_defaults(func=start_recursive_analysis) self._build_args(optionlist=ARGS_RECURSIVE_ANALYSIS, diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index a815cd5f3a9..b1df7e98ba9 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -15,6 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration """ + from freqtrade.persistence import FtNoDBContext from freqtrade.plugins.pairlistmanager import PairListManager config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) @@ -24,11 +25,12 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: if not quote_currencies: quote_currencies = [config.get('stake_currency')] results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist + with FtNoDBContext(): + for curr in quote_currencies: + config['stake_currency'] = curr + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist for curr, pairlist in results.items(): if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False): diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e5e4d28a00a..6fff02648e7 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -68,8 +68,10 @@ def load_config(self) -> Dict[str, Any]: config: Config = load_from_files(self.args.get("config", [])) # Load environment variables - env_data = enironment_vars_to_dict() - config = deep_merge_dicts(env_data, config) + from freqtrade.commands.arguments import NO_CONF_ALLOWED + if self.args.get('command') not in NO_CONF_ALLOWED: + env_data = enironment_vars_to_dict() + config = deep_merge_dicts(env_data, config) # Normalize config if 'internals' not in config: diff --git a/freqtrade/configuration/environment_vars.py b/freqtrade/configuration/environment_vars.py index c5efd45b7df..b59b10fa295 100644 --- a/freqtrade/configuration/environment_vars.py +++ b/freqtrade/configuration/environment_vars.py @@ -9,7 +9,7 @@ logger = logging.getLogger(__name__) -def get_var_typed(val): +def _get_var_typed(val): try: return int(val) except ValueError: @@ -24,7 +24,7 @@ def get_var_typed(val): return val -def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]: +def _flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, Any]: """ Environment variables must be prefixed with FREQTRADE. FREQTRADE__{section}__{key} @@ -40,7 +40,7 @@ def flat_vars_to_nested_dict(env_dict: Dict[str, Any], prefix: str) -> Dict[str, logger.info(f"Loading variable '{env_var}'") key = env_var.replace(prefix, '') for k in reversed(key.split('__')): - val = {k.lower(): get_var_typed(val) + val = {k.lower(): _get_var_typed(val) if not isinstance(val, dict) and k not in no_convert else val} relevant_vars = deep_merge_dicts(val, relevant_vars) return relevant_vars @@ -52,4 +52,4 @@ def enironment_vars_to_dict() -> Dict[str, Any]: Relevant variables must follow the FREQTRADE__{section}__{key} pattern :return: Nested dict based on available and relevant variables. """ - return flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX) + return _flat_vars_to_nested_dict(os.environ.copy(), ENV_VAR_PREFIX) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 64b6ae593cc..b0a8485b836 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1453,11 +1453,11 @@ def replace_order(self, order: Dict, order_obj: Optional[Order], trade: Trade) - # New candle proposed_rate = self.exchange.get_rate( trade.pair, side='entry', is_short=trade.is_short, refresh=True) - adjusted_entry_price = strategy_safe_wrapper(self.strategy.adjust_entry_price, - default_retval=order_obj.price)( + adjusted_entry_price = strategy_safe_wrapper( + self.strategy.adjust_entry_price, default_retval=order_obj.safe_placement_price)( trade=trade, order=order_obj, pair=trade.pair, current_time=datetime.now(timezone.utc), proposed_rate=proposed_rate, - current_order_rate=order_obj.safe_price, entry_tag=trade.enter_tag, + current_order_rate=order_obj.safe_placement_price, entry_tag=trade.enter_tag, side=trade.trade_direction) replacing = True @@ -1465,7 +1465,7 @@ def replace_order(self, order: Dict, order_obj: Optional[Order], trade: Trade) - if not adjusted_entry_price: replacing = False cancel_reason = constants.CANCEL_REASON['USER_CANCEL'] - if order_obj.price != adjusted_entry_price: + if order_obj.safe_placement_price != adjusted_entry_price: # cancel existing order if new price is supplied or None res = self.handle_cancel_enter(trade, order, order_obj, cancel_reason, replacing=replacing) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index ce37a0dcc43..137e51de5a5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -33,7 +33,8 @@ show_backtest_results, store_backtest_analysis_results, store_backtest_stats) -from freqtrade.persistence import LocalTrade, Order, PairLocks, Trade +from freqtrade.persistence import (LocalTrade, Order, PairLocks, Trade, disable_database_use, + enable_database_use) from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -177,8 +178,7 @@ def _validate_pairlists_for_backtesting(self): @staticmethod def cleanup(): LoggingMixin.show_output = True - PairLocks.use_db = True - Trade.use_db = True + enable_database_use() def init_backtest_detail(self) -> None: # Load detail timeframe if specified @@ -325,9 +325,7 @@ def load_bt_data_detail(self) -> None: self.futures_data = {} def disable_database_use(self): - PairLocks.use_db = False - PairLocks.timeframe = self.timeframe - Trade.use_db = False + disable_database_use(self.timeframe) def prepare_backtest(self, enable_protections): """ diff --git a/freqtrade/optimize/base_analysis.py b/freqtrade/optimize/base_analysis.py index 190ac882f16..eb0a5e00239 100644 --- a/freqtrade/optimize/base_analysis.py +++ b/freqtrade/optimize/base_analysis.py @@ -54,7 +54,7 @@ def fill_full_varholder(self): self.full_varHolder.from_dt = parsed_timerange.startdt if parsed_timerange.stopdt is None: - self.full_varHolder.to_dt = datetime.utcnow() + self.full_varHolder.to_dt = datetime.now(timezone.utc) else: self.full_varHolder.to_dt = parsed_timerange.stopdt diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 4cf7aa455fa..6205174a764 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -4,3 +4,5 @@ from freqtrade.persistence.models import init_db from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.persistence.trade_model import LocalTrade, Order, Trade +from freqtrade.persistence.usedb_context import (FtNoDBContext, disable_database_use, + enable_database_use) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 5f5e99ed80e..ea0b640d5c9 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -106,6 +106,11 @@ def order_filled_utc(self) -> Optional[datetime]: def safe_amount(self) -> float: return self.amount or self.ft_amount + @property + def safe_placement_price(self) -> float: + """Price at which the order was placed""" + return self.price or self.stop_price or self.ft_price + @property def safe_price(self) -> float: return self.average or self.price or self.stop_price or self.ft_price diff --git a/freqtrade/persistence/usedb_context.py b/freqtrade/persistence/usedb_context.py new file mode 100644 index 00000000000..6fffd2fb598 --- /dev/null +++ b/freqtrade/persistence/usedb_context.py @@ -0,0 +1,33 @@ + +from freqtrade.persistence.pairlock_middleware import PairLocks +from freqtrade.persistence.trade_model import Trade + + +def disable_database_use(timeframe: str) -> None: + """ + Disable database usage for PairLocks and Trade models. + Used for backtesting, and some other utility commands. + """ + PairLocks.use_db = False + PairLocks.timeframe = timeframe + Trade.use_db = False + + +def enable_database_use() -> None: + """ + Cleanup function to restore database usage. + """ + PairLocks.use_db = True + PairLocks.timeframe = '' + Trade.use_db = True + + +class FtNoDBContext: + def __init__(self, timeframe: str = ''): + self.timeframe = timeframe + + def __enter__(self): + disable_database_use(self.timeframe) + + def __exit__(self, exc_type, exc_val, exc_tb): + enable_database_use() diff --git a/freqtrade/rpc/api_server/api_auth.py b/freqtrade/rpc/api_server/api_auth.py index 71f1145a9f6..dd47491002a 100644 --- a/freqtrade/rpc/api_server/api_auth.py +++ b/freqtrade/rpc/api_server/api_auth.py @@ -1,6 +1,6 @@ import logging import secrets -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, Union import jwt @@ -88,14 +88,14 @@ async def validate_ws_token( def create_token(data: dict, secret_key: str, token_type: str = "access") -> str: to_encode = data.copy() if token_type == "access": - expire = datetime.utcnow() + timedelta(minutes=15) + expire = datetime.now(timezone.utc) + timedelta(minutes=15) elif token_type == "refresh": - expire = datetime.utcnow() + timedelta(days=30) + expire = datetime.now(timezone.utc) + timedelta(days=30) else: raise ValueError() to_encode.update({ "exp": expire, - "iat": datetime.utcnow(), + "iat": datetime.now(timezone.utc), "type": token_type, }) encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM) diff --git a/freqtrade/rpc/api_server/api_background_tasks.py b/freqtrade/rpc/api_server/api_background_tasks.py index c13fa31e42a..04e98c609cd 100644 --- a/freqtrade/rpc/api_server/api_background_tasks.py +++ b/freqtrade/rpc/api_server/api_background_tasks.py @@ -7,6 +7,7 @@ from freqtrade.constants import Config from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException +from freqtrade.persistence import FtNoDBContext from freqtrade.rpc.api_server.api_schemas import (BackgroundTaskStatus, BgJobStarted, ExchangeModePayloadMixin, PairListsPayload, PairListsResponse, WhitelistEvaluateResponse) @@ -57,16 +58,16 @@ def __run_pairlist(job_id: str, config_loc: Config): ApiBG.jobs[job_id]['is_running'] = True from freqtrade.plugins.pairlistmanager import PairListManager - - exchange = get_exchange(config_loc) - pairlists = PairListManager(exchange, config_loc) - pairlists.refresh_pairlist() - ApiBG.jobs[job_id]['result'] = { - 'method': pairlists.name_list, - 'length': len(pairlists.whitelist), - 'whitelist': pairlists.whitelist - } - ApiBG.jobs[job_id]['status'] = 'success' + with FtNoDBContext(): + exchange = get_exchange(config_loc) + pairlists = PairListManager(exchange, config_loc) + pairlists.refresh_pairlist() + ApiBG.jobs[job_id]['result'] = { + 'method': pairlists.name_list, + 'length': len(pairlists.whitelist), + 'whitelist': pairlists.whitelist + } + ApiBG.jobs[job_id]['status'] = 'success' except (OperationalException, Exception) as e: logger.exception(e) ApiBG.jobs[job_id]['error'] = str(e) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index bd846eb9012..675e4a7c150 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -1388,7 +1388,8 @@ def advise_entry(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ logger.debug(f"Populating enter signals for pair {metadata.get('pair')}.") - + # Initialize column to work around Pandas bug #56503. + dataframe.loc[:, 'enter_tag'] = '' df = self.populate_entry_trend(dataframe, metadata) if 'enter_long' not in df.columns: df = df.rename({'buy': 'enter_long', 'buy_tag': 'enter_tag'}, axis='columns') @@ -1404,6 +1405,8 @@ def advise_exit(self, dataframe: DataFrame, metadata: dict) -> DataFrame: currently traded pair :return: DataFrame with exit column """ + # Initialize column to work around Pandas bug #56503. + dataframe.loc[:, 'exit_tag'] = '' logger.debug(f"Populating exit signals for pair {metadata.get('pair')}.") df = self.populate_exit_trend(dataframe, metadata) if 'exit_long' not in df.columns: diff --git a/requirements-dev.txt b/requirements-dev.txt index 086379d6c6e..7da5d559c0f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ -r docs/requirements-docs.txt coveralls==3.3.1 -ruff==0.1.11 +ruff==0.1.13 mypy==1.8.0 pre-commit==3.6.0 pytest==7.4.4 @@ -21,7 +21,7 @@ isort==5.13.2 time-machine==2.13.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.14.0 +nbconvert==7.14.1 # mypy types types-cachetools==5.3.0.7 diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index 55a09e6f983..fa5e9f01414 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -2,10 +2,10 @@ -r requirements-freqai.txt # Required for freqai-rl -torch==2.1.2 +torch==2.1.2; python_version < '3.12' #until these branches will be released we can use this -gymnasium==0.29.1 -stable_baselines3==2.2.1 -sb3_contrib>=2.0.0a9 +gymnasium==0.29.1; python_version < '3.12' +stable_baselines3==2.2.1; python_version < '3.12' +sb3_contrib>=2.0.0a9; python_version < '3.12' # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.66.1 diff --git a/requirements-freqai.txt b/requirements-freqai.txt index 88f3da0a974..17087d8ca17 100644 --- a/requirements-freqai.txt +++ b/requirements-freqai.txt @@ -5,7 +5,7 @@ # Required for freqai scikit-learn==1.3.2 joblib==1.3.2 -catboost==1.2.2; 'arm' not in platform_machine +catboost==1.2.2; 'arm' not in platform_machine and python_version < '3.12' lightgbm==4.2.0 xgboost==2.0.3 tensorboard==2.15.1 diff --git a/requirements.txt b/requirements.txt index adc3ef5a3b0..9380965792e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b finta>=1.3 ta>=0.10.1 -ccxt==4.2.9 +ccxt==4.2.14 cryptography==41.0.7 aiohttp==3.9.1 SQLAlchemy==2.0.25 @@ -20,7 +20,7 @@ TA-Lib==0.4.28 technical==1.4.2 tabulate==0.9.0 pycoingecko==3.1.0 -jinja2==3.1.2 +jinja2==3.1.3 tables==3.9.1 joblib==1.3.2 rich==13.7.0 @@ -38,7 +38,7 @@ orjson==3.9.10 sdnotify==0.3.2 # API Server -fastapi==0.108.0 +fastapi==0.109.0 pydantic==2.5.3 uvicorn==0.25.0 pyjwt==2.8.0 diff --git a/tests/freqai/conftest.py b/tests/freqai/conftest.py index 57ba3f64bd7..81d72d92a88 100644 --- a/tests/freqai/conftest.py +++ b/tests/freqai/conftest.py @@ -1,4 +1,5 @@ import platform +import sys from copy import deepcopy from pathlib import Path from typing import Any, Dict @@ -15,6 +16,10 @@ from tests.conftest import get_patched_exchange +def is_py12() -> bool: + return sys.version_info >= (3, 12) + + def is_mac() -> bool: machine = platform.system() return "Darwin" in machine @@ -31,7 +36,7 @@ def patch_torch_initlogs(mocker) -> None: module_name = 'torch' mocked_module = types.ModuleType(module_name) sys.modules[module_name] = mocked_module - else: + elif not is_py12(): mocker.patch("torch._logging._init_logs") diff --git a/tests/freqai/test_freqai_interface.py b/tests/freqai/test_freqai_interface.py index cc5a9b3260c..58648d97f3c 100644 --- a/tests/freqai/test_freqai_interface.py +++ b/tests/freqai/test_freqai_interface.py @@ -1,7 +1,6 @@ import logging import platform import shutil -import sys from pathlib import Path from unittest.mock import MagicMock @@ -16,24 +15,24 @@ from freqtrade.persistence import Trade from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_has_re -from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, make_rl_config, +from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, is_py12, make_rl_config, mock_pytorch_mlp_model_training_parameters) -def is_py12() -> bool: - return sys.version_info >= (3, 12) - - def is_arm() -> bool: machine = platform.machine() return "arm" in machine or "aarch64" in machine def can_run_model(model: str) -> None: + is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model + + if is_py12() and ("Catboost" in model or is_pytorch_model): + pytest.skip("Model not supported on python 3.12 yet.") + if is_arm() and "Catboost" in model: pytest.skip("CatBoost is not supported on ARM.") - is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model if is_pytorch_model and is_mac() and not is_arm(): pytest.skip("Reinforcement learning / PyTorch module not available on intel based Mac OS.") diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index ca68724c945..87e92071f1b 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -734,7 +734,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'min_rate': [0.10370188, 0.10300000000000001], 'max_rate': [0.10501, 0.1038888], 'is_open': [False, False], - 'enter_tag': [None, None], + 'enter_tag': ['', ''], "leverage": [1.0, 1.0], "is_short": [False, False], 'open_timestamp': [1517251200000, 1517283000000], diff --git a/tests/optimize/test_backtesting_adjust_position.py b/tests/optimize/test_backtesting_adjust_position.py index 9b40b3a9d0a..56b04b3fd71 100644 --- a/tests/optimize/test_backtesting_adjust_position.py +++ b/tests/optimize/test_backtesting_adjust_position.py @@ -72,7 +72,7 @@ def test_backtest_position_adjustment(default_conf, fee, mocker, testdatadir) -> 'min_rate': [0.10370188, 0.10300000000000001], 'max_rate': [0.10481985, 0.1038888], 'is_open': [False, False], - 'enter_tag': [None, None], + 'enter_tag': ['', ''], 'leverage': [1.0, 1.0], 'is_short': [False, False], 'open_timestamp': [1517251200000, 1517283000000], diff --git a/tests/persistence/test_db_context.py b/tests/persistence/test_db_context.py new file mode 100644 index 00000000000..690006219e4 --- /dev/null +++ b/tests/persistence/test_db_context.py @@ -0,0 +1,24 @@ +import pytest + +from freqtrade.persistence import FtNoDBContext, PairLocks, Trade + + +@pytest.mark.parametrize('timeframe', ['', '5m', '1d']) +def test_FtNoDBContext(timeframe): + PairLocks.timeframe = '' + assert Trade.use_db is True + assert PairLocks.use_db is True + assert PairLocks.timeframe == '' + + with FtNoDBContext(timeframe): + assert Trade.use_db is False + assert PairLocks.use_db is False + assert PairLocks.timeframe == timeframe + + with FtNoDBContext(): + assert Trade.use_db is False + assert PairLocks.use_db is False + assert PairLocks.timeframe == '' + + assert Trade.use_db is True + assert PairLocks.use_db is True diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 9957ebdb033..81d01acad30 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1603,9 +1603,9 @@ def test_api_pair_history(botclient, mocker): assert 'data' in result data = result['data'] assert len(data) == 289 - # analyed DF has 28 columns - assert len(result['columns']) == 28 - assert len(data[0]) == 28 + # analyed DF has 30 columns + assert len(result['columns']) == 30 + assert len(data[0]) == 30 date_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'date'][0] rsi_col_idx = [idx for idx, c in enumerate(result['columns']) if c == 'rsi'][0] diff --git a/tests/rpc/test_rpc_emc.py b/tests/rpc/test_rpc_emc.py index 26512e30b14..4cfa3e9dbd8 100644 --- a/tests/rpc/test_rpc_emc.py +++ b/tests/rpc/test_rpc_emc.py @@ -2,7 +2,6 @@ Unit test file for rpc/external_message_consumer.py """ import asyncio -import functools import logging from datetime import datetime, timezone from unittest.mock import MagicMock @@ -302,19 +301,16 @@ async def test_emc_receive_messages_valid(default_conf, caplog, mocker): dp = DataProvider(default_conf, None, None, None) emc = ExternalMessageConsumer(default_conf, dp) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running - class TestChannel: async def recv(self, *args, **kwargs): + emc._running = False return {"type": "whitelist", "data": ["BTC/USDT"]} async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + emc._running = True await emc._receive_messages(TestChannel(), test_producer, lock) assert log_has_re(r"Received message of type `whitelist`.+", caplog) @@ -349,19 +345,16 @@ async def test_emc_receive_messages_invalid(default_conf, caplog, mocker): dp = DataProvider(default_conf, None, None, None) emc = ExternalMessageConsumer(default_conf, dp) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running - class TestChannel: async def recv(self, *args, **kwargs): + emc._running = False return {"type": ["BTC/USDT"]} async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + emc._running = True await emc._receive_messages(TestChannel(), test_producer, lock) assert log_has_re(r"Invalid message from.+", caplog) @@ -396,8 +389,8 @@ async def test_emc_receive_messages_timeout(default_conf, caplog, mocker): dp = DataProvider(default_conf, None, None, None) emc = ExternalMessageConsumer(default_conf, dp) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running + def change_running(): + emc._running = not emc._running class TestChannel: async def recv(self, *args, **kwargs): @@ -407,8 +400,7 @@ async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + change_running() with pytest.raises(asyncio.TimeoutError): await emc._receive_messages(TestChannel(), test_producer, lock) @@ -447,19 +439,16 @@ async def test_emc_receive_messages_handle_error(default_conf, caplog, mocker): emc.handle_producer_message = MagicMock(side_effect=Exception) - loop = asyncio.get_event_loop() - def change_running(emc): emc._running = not emc._running - class TestChannel: async def recv(self, *args, **kwargs): + emc._running = False return {"type": "whitelist", "data": ["BTC/USDT"]} async def ping(self, *args, **kwargs): return asyncio.Future() try: - change_running(emc) - loop.call_soon(functools.partial(change_running, emc=emc)) + emc._running = True await emc._receive_messages(TestChannel(), test_producer, lock) assert log_has_re(r"Error handling producer message.+", caplog) diff --git a/tests/strategy/strats/strategy_test_v3.py b/tests/strategy/strats/strategy_test_v3.py index 571427fb150..83c7353ce6a 100644 --- a/tests/strategy/strats/strategy_test_v3.py +++ b/tests/strategy/strats/strategy_test_v3.py @@ -152,7 +152,7 @@ def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFram ( qtpylib.crossed_below(dataframe['rsi'], self.sell_rsi.value) ), - 'enter_short'] = 1 + ('enter_short', 'enter_tag')] = (1, 'short_Tag') return dataframe @@ -176,7 +176,7 @@ def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame ( qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value) ), - 'exit_short'] = 1 + ('exit_short', 'exit_tag')] = (1, 'short_Tag') return dataframe diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 226bbc7ae86..790f5d25501 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -105,7 +105,7 @@ def test_returns_latest_signal(ohlcv_history): _STRATEGY.config['trading_mode'] = 'spot' -def test_analyze_pair_empty(default_conf, mocker, caplog, ohlcv_history): +def test_analyze_pair_empty(mocker, caplog, ohlcv_history): mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history) mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', @@ -1019,3 +1019,30 @@ def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): StrategyResolver.load_strategy(default_conf) assert log_has("Invalid parameter file format.", caplog) + + +@pytest.mark.parametrize('function,raises', [ + ('populate_entry_trend', True), + ('advise_entry', False), + ('populate_exit_trend', True), + ('advise_exit', False), +]) +def test_pandas_warning_direct(ohlcv_history, function, raises): + + df = _STRATEGY.populate_indicators(ohlcv_history, {'pair': 'ETH/BTC'}) + if raises: + with pytest.warns(FutureWarning): + # Test for Future warning + # FutureWarning: Setting an item of incompatible dtype is + # deprecated and will raise in a future error of pandas + # https://github.com/pandas-dev/pandas/issues/56503 + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) + else: + getattr(_STRATEGY, function)(df, {'pair': 'ETH/BTC'}) + + +def test_pandas_warning_through_analyze_pair(ohlcv_history, mocker, recwarn): + + mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history) + _STRATEGY.analyze_pair('ETH/BTC') + assert len(recwarn) == 0 diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 6472faf3784..71585cac385 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -15,7 +15,7 @@ process_deprecated_setting, process_removed_setting, process_temporary_deprecated_settings) -from freqtrade.configuration.environment_vars import flat_vars_to_nested_dict +from freqtrade.configuration.environment_vars import _flat_vars_to_nested_dict from freqtrade.configuration.load_config import (load_config_file, load_file, load_from_files, log_config_error_range) from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL, ENV_VAR_PREFIX @@ -1419,7 +1419,7 @@ def test_flat_vars_to_nested_dict(caplog): 'chat_id': '2151' } } - res = flat_vars_to_nested_dict(test_args, ENV_VAR_PREFIX) + res = _flat_vars_to_nested_dict(test_args, ENV_VAR_PREFIX) assert res == expected assert log_has("Loading variable 'FREQTRADE__EXCHANGE__SOME_SETTING'", caplog) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index fea30fc4ca2..e61d5804d14 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -627,15 +627,16 @@ def test_process_exchange_failures(default_conf_usdt, ticker_usdt, mocker) -> No mocker.patch.multiple( EXMS, fetch_ticker=ticker_usdt, - create_order=MagicMock(side_effect=TemporaryError) + reload_markets=MagicMock(side_effect=TemporaryError), + create_order=MagicMock(side_effect=TemporaryError), ) - sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) + sleep_mock = mocker.patch('time.sleep') worker = Worker(args=None, config=default_conf_usdt) patch_get_signal(worker.freqtrade) worker._process_running() - assert sleep_mock.has_calls() + assert sleep_mock.called is True def test_process_operational_exception(default_conf_usdt, ticker_usdt, mocker) -> None: