Merge branch 'test' into freqtrade-develop

.github/workflows/binance-lev-tier-update.yml

@@ -19,7 +19,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Install ccxt
       run: pip install ccxt

.github/workflows/ci.yml

@@ -334,7 +334,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: pre-commit dependencies
       run: |
@@ -348,7 +348,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version: "3.12"
     - uses: pre-commit/[email protected]
 
   docs-check:
@@ -363,7 +363,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Documentation build
       run: |
@@ -389,7 +389,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Cache_dependencies
       uses: actions/cache@v4
@@ -471,7 +471,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Build distribution
       run: |
@@ -542,7 +542,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
     - name: Extract branch name
       id: extract-branch

.github/workflows/pre-commit-update.yml

@@ -17,7 +17,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.12"
 
 
     - name: Install pre-commit

docs/data-download.md

@@ -24,10 +24,10 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [--days INT] [--new-pairs-days INT]
                                [--include-inactive-pairs]
                                [--timerange TIMERANGE] [--dl-trades]
-                               [--exchange EXCHANGE]
+                               [--convert] [--exchange EXCHANGE]
                                [-t TIMEFRAMES [TIMEFRAMES ...]] [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5,feather,parquet}]
-                               [--data-format-trades {json,jsongz,hdf5,feather}]
+                               [--data-format-trades {json,jsongz,hdf5,feather,parquet}]
                                [--trading-mode {spot,margin,futures}]
                                [--prepend]
 
@@ -48,6 +48,11 @@ options:
   --dl-trades           Download trades instead of OHLCV data. The bot will
                         resample trades to the desired timeframe as specified
                         as --timeframes/-t.
+  --convert             Convert downloaded trades to OHLCV data. Only
+                        applicable in combination with `--dl-trades`. Will be
+                        automatic for exchanges which don't have historic
+                        OHLCV (e.g. Kraken). If not provided, use `trades-to-
+                        ohlcv` to convert trades data to OHLCV data.
   --exchange EXCHANGE   Exchange name. Only valid if no config is provided.
   -t TIMEFRAMES [TIMEFRAMES ...], --timeframes TIMEFRAMES [TIMEFRAMES ...]
                         Specify which tickers to download. Space-separated
@@ -57,7 +62,7 @@ options:
   --data-format-ohlcv {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded candle (OHLCV) data.
                         (default: `feather`).
-  --data-format-trades {json,jsongz,hdf5,feather}
+  --data-format-trades {json,jsongz,hdf5,feather,parquet}
                         Storage format for downloaded trades data. (default:
                         `feather`).
   --trading-mode {spot,margin,futures}, --tradingmode {spot,margin,futures}
@@ -471,15 +476,20 @@ ETH/USDT    5m, 15m, 30m, 1h, 2h, 4h
 
 ## Trades (tick) data
 
-By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API.
+By default, `download-data` sub-command downloads Candles (OHLCV) data. Most exchanges also provide historic trade-data via their API.
 This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes.
 
-Since this data is large by default, the files use the feather fileformat by default. They are stored in your data-directory with the naming convention of `<pair>-trades.feather` (`ETH_BTC-trades.feather`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
+Since this data is large by default, the files use the feather file format by default. They are stored in your data-directory with the naming convention of `<pair>-trades.feather` (`ETH_BTC-trades.feather`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository.
+
+To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades.
+If `--convert` is also provided, the resample step will happen automatically and overwrite eventually existing OHLCV data for the given pair/timeframe combinations.
 
-To use this mode, simply add `--dl-trades` to your call. This will swap the download method to download trades, and resamples the data locally.
+!!! Warning "Do not use"
+    You should not use this unless you're a kraken user (Kraken does not provide historic OHLCV data).  
+    Most other exchanges provide OHLCV data with sufficient history, so downloading multiple timeframes through that method will still proof to be a lot faster than downloading trades data.
 
-!!! Warning "do not use"
-    You should not use this unless you're a kraken user. Most other exchanges provide OHLCV data with sufficient history.
+!!! Note "Kraken user"
+    Kraken users should read [this](exchanges.md#historic-kraken-data) before starting to download data.
 
 Example call:
 
@@ -490,12 +500,6 @@ freqtrade download-data --exchange kraken --pairs XRP/EUR ETH/EUR --days 20 --dl
 !!! Note
     While this method uses async calls, it will be slow, since it requires the result of the previous call to generate the next request to the exchange.
 
-!!! Warning
-    The historic trades are not available during Freqtrade dry-run and live trade modes because all exchanges tested provide this data with a delay of few 100 candles, so it's not suitable for real-time trading.
-
-!!! Note "Kraken user"
-    Kraken users should read [this](exchanges.md#historic-kraken-data) before starting to download data.
-
 ## Next step
 
-Great, you now have backtest data downloaded, so you can now start [backtesting](backtesting.md) your strategy.
+Great, you now have some data downloaded, so you can now start [backtesting](backtesting.md) your strategy.

docs/installation.md

@@ -286,7 +286,7 @@ cd freqtrade
 #### Freqtrade install: Conda Environment
 
 ```bash
-conda create --name freqtrade python=3.11
+conda create --name freqtrade python=3.12
 ```
 
 !!! Note "Creating Conda Environment"

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.6
 mkdocs==1.6.0
-mkdocs-material==9.5.22
+mkdocs-material==9.5.23
 mdx_truly_sane_lists==1.3
 pymdown-extensions==10.8.1
 jinja2==3.1.4

docs/windows_installation.md

@@ -24,7 +24,7 @@ git clone https://github.com/freqtrade/freqtrade.git
 
 Install ta-lib according to the [ta-lib documentation](https://github.com/TA-Lib/ta-lib-python#windows).
 
-As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10 and 3.11) and for 64bit Windows.
+As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), Freqtrade provides these dependencies (in the binary wheel format) for the latest 3 Python versions (3.9, 3.10, 3.11 and 3.12) and for 64bit Windows.
 These Wheels are also used by CI running on windows, and are therefore tested together with freqtrade.
 
 Other versions must be downloaded from the above link.

freqtrade/commands/arguments.py

@@ -142,6 +142,7 @@
     "include_inactive",
     "timerange",
     "download_trades",
+    "convert_trades",
     "exchange",
     "timeframes",
     "erase",

freqtrade/commands/cli_options.py

@@ -450,6 +450,14 @@ def __init__(self, *args, **kwargs):
         "desired timeframe as specified as --timeframes/-t.",
         action="store_true",
     ),
+    "convert_trades": Arg(
+        "--convert",
+        help="Convert downloaded trades to OHLCV data. Only applicable in combination with "
+        "`--dl-trades`. "
+        "Will be automatic for exchanges which don't have historic OHLCV (e.g. Kraken). "
+        "If not provided, use `trades-to-ohlcv` to convert trades data to OHLCV data.",
+        action="store_true",
+    ),
     "format_from_trades": Arg(
         "--format-from",
         help="Source format for data conversion.",

freqtrade/configuration/configuration.py

@@ -370,6 +370,7 @@ def _process_plot_options(self, config: Config) -> None:
             ("days", "Detected --days: {}"),
             ("include_inactive", "Detected --include-inactive-pairs: {}"),
             ("download_trades", "Detected --dl-trades: {}"),
+            ("convert_trades", "Detected --convert: {} - Converting Trade data to OHCV {}"),
             ("dataformat_ohlcv", 'Using "{}" to store OHLCV data.'),
             ("dataformat_trades", 'Using "{}" to store trades data.'),
             ("show_timerange", "Detected --show-timerange"),

freqtrade/data/history/history_utils.py

@@ -629,17 +629,20 @@ def download_data_main(config: Config) -> None:
                 trading_mode=config.get("trading_mode", TradingMode.SPOT),
             )
 
-            # Convert downloaded trade data to different timeframes
-            convert_trades_to_ohlcv(
-                pairs=expanded_pairs,
-                timeframes=config["timeframes"],
-                datadir=config["datadir"],
-                timerange=timerange,
-                erase=bool(config.get("erase")),
-                data_format_ohlcv=config["dataformat_ohlcv"],
-                data_format_trades=config["dataformat_trades"],
-                candle_type=config.get("candle_type_def", CandleType.SPOT),
-            )
+            if config.get("convert_trades") or not exchange.get_option("ohlcv_has_history", True):
+                # Convert downloaded trade data to different timeframes
+                # Only auto-convert for exchanges without historic klines
+
+                convert_trades_to_ohlcv(
+                    pairs=expanded_pairs,
+                    timeframes=config["timeframes"],
+                    datadir=config["datadir"],
+                    timerange=timerange,
+                    erase=bool(config.get("erase")),
+                    data_format_ohlcv=config["dataformat_ohlcv"],
+                    data_format_trades=config["dataformat_trades"],
+                    candle_type=config.get("candle_type_def", CandleType.SPOT),
+                )
         else:
             if not exchange.get_option("ohlcv_has_history", True):
                 raise OperationalException(

freqtrade/exchange/binance.py

@@ -206,24 +206,13 @@ def dry_run_liquidation_price(
                 "Freqtrade only supports isolated futures for leverage trading"
             )
 
-    @retrier
     def load_leverage_tiers(self) -> Dict[str, List[Dict]]:
         if self.trading_mode == TradingMode.FUTURES:
             if self._config["dry_run"]:
                 leverage_tiers_path = Path(__file__).parent / "binance_leverage_tiers.json"
                 with leverage_tiers_path.open() as json_file:
                     return json_load(json_file)
             else:
-                try:
-                    return self._api.fetch_leverage_tiers()
-                except ccxt.DDoSProtection as e:
-                    raise DDosProtection(e) from e
-                except (ccxt.OperationFailed, ccxt.ExchangeError) as e:
-                    raise TemporaryError(
-                        f"Could not fetch leverage amounts due to"
-                        f"{e.__class__.__name__}. Message: {e}"
-                    ) from e
-                except ccxt.BaseError as e:
-                    raise OperationalException(e) from e
+                return self.get_leverage_tiers()
         else:
             return {}

freqtrade/exchange/bingx.py

@@ -17,4 +17,6 @@ class Bingx(Exchange):
 
     _ft_has: Dict = {
         "ohlcv_candle_limit": 1000,
+        "stoploss_on_exchange": False,
+        "stoploss_order_types": {"limit": "limit", "market": "market"},
     }

freqtrade/exchange/bybit.py

@@ -250,38 +250,19 @@ def fetch_order(self, order_id: str, pair: str, params: Optional[Dict] = None) -
     @retrier
     def get_leverage_tiers(self) -> Dict[str, List[Dict]]:
         """
-        Temporary workaround for https://github.com/freqtrade/freqtrade/issues/10196
-        should be removed or updated once https://github.com/ccxt/ccxt/issues/22448 is fixed.
+        Cache leverage tiers for 1 day, since they are not expected to change often, and
+        bybit requires pagination to fetch all tiers.
         """
 
         # Load cached tiers
-        tiers_cached = self.load_cached_leverage_tiers(self._config["stake_currency"])
+        tiers_cached = self.load_cached_leverage_tiers(
+            self._config["stake_currency"], timedelta(days=1)
+        )
         if tiers_cached:
-            tiers = tiers_cached
-            return tiers
+            return tiers_cached
 
         # Fetch tiers from exchange
-
-        symbols = self._api.market_symbols([])
-
-        def parse_resp(response):
-            result = self._api.safe_dict(response, "result", {})
-            data = self._api.safe_list(result, "list", [])
-            return self._api.parse_leverage_tiers(data, symbols, "symbol")
-
-        params = {
-            "category": "linear",
-        }
-        tiers = {}
-        # 20 pairs ... should be sufficient assuming 30 pairs per page
-        # Aimed to avoid a potential infinite loop
-        for _ in range(20):
-            # Fetch from private endpoint
-            response = self._api.publicGetV5MarketRiskLimit(params)
-            tiers = tiers | parse_resp(response)
-            if (cursor := response["result"]["nextPageCursor"]) == "":
-                break
-            params.update({"cursor": cursor})
+        tiers = super().get_leverage_tiers()
 
         self.cache_leverage_tiers(tiers, self._config["stake_currency"])
         return tiers

freqtrade/exchange/exchange.py

@@ -2868,15 +2868,24 @@ def cache_leverage_tiers(self, tiers: Dict[str, List[Dict]], stake_currency: str
         }
         file_dump_json(filename, data)
 
-    def load_cached_leverage_tiers(self, stake_currency: str) -> Optional[Dict[str, List[Dict]]]:
+    def load_cached_leverage_tiers(
+        self, stake_currency: str, cache_time: Optional[timedelta] = None
+    ) -> Optional[Dict[str, List[Dict]]]:
+        """
+        Load cached leverage tiers from disk
+        :param cache_time: The maximum age of the cache before it is considered outdated
+        """
+        if not cache_time:
+            # Default to 4 weeks
+            cache_time = timedelta(weeks=4)
         filename = self._config["datadir"] / "futures" / f"leverage_tiers_{stake_currency}.json"
         if filename.is_file():
             try:
                 tiers = file_load_json(filename)
                 updated = tiers.get("updated")
                 if updated:
                     updated_dt = parser.parse(updated)
-                    if updated_dt < datetime.now(timezone.utc) - timedelta(weeks=4):
+                    if updated_dt < datetime.now(timezone.utc) - cache_time:
                         logger.info("Cached leverage tiers are outdated. Will update.")
                         return None
                 return tiers["data"]

freqtrade/freqai/prediction_models/CatboostClassifier.py

@@ -1,5 +1,4 @@
 import logging
-import sys
 from pathlib import Path
 from typing import Any, Dict
 
@@ -57,8 +56,6 @@ def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
             X=train_data,
             eval_set=test_data,
             init_model=init_model,
-            log_cout=sys.stdout,
-            log_cerr=sys.stderr,
         )
 
         return cbr

freqtrade/freqai/prediction_models/CatboostClassifierMultiTarget.py

@@ -1,5 +1,4 @@
 import logging
-import sys
 from pathlib import Path
 from typing import Any, Dict
 
@@ -68,8 +67,6 @@ def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
                 {
                     "eval_set": eval_sets[i],
                     "init_model": init_models[i],
-                    "log_cout": sys.stdout,
-                    "log_cerr": sys.stderr,
                 }
             )
 

freqtrade/freqai/prediction_models/CatboostRegressor.py

@@ -1,5 +1,4 @@
 import logging
-import sys
 from pathlib import Path
 from typing import Any, Dict
 
@@ -56,8 +55,6 @@ def fit(self, data_dictionary: Dict, dk: FreqaiDataKitchen, **kwargs) -> Any:
             X=train_data,
             eval_set=test_data,
             init_model=init_model,
-            log_cout=sys.stdout,
-            log_cerr=sys.stderr,
         )
 
         return model