From 6b6b669ee781f4a9823c38432baee2b08914d871 Mon Sep 17 00:00:00 2001 From: herklos Date: Tue, 8 Jun 2021 22:47:29 +0200 Subject: [PATCH 01/44] [Trading][Exchanges] Add FTX --- Trading/Exchange/ftx/__init__.py | 1 + Trading/Exchange/ftx/ftx_exchange.py | 70 ++++++++++++++++++++++++++ Trading/Exchange/ftx/metadata.json | 6 +++ Trading/Exchange/ftx/resources/ftx.md | 1 + Trading/Exchange/ftx/tests/__init__.py | 15 ++++++ 5 files changed, 93 insertions(+) create mode 100644 Trading/Exchange/ftx/__init__.py create mode 100644 Trading/Exchange/ftx/ftx_exchange.py create mode 100644 Trading/Exchange/ftx/metadata.json create mode 100644 Trading/Exchange/ftx/resources/ftx.md create mode 100644 Trading/Exchange/ftx/tests/__init__.py diff --git a/Trading/Exchange/ftx/__init__.py b/Trading/Exchange/ftx/__init__.py new file mode 100644 index 000000000..8ca0bb9bd --- /dev/null +++ b/Trading/Exchange/ftx/__init__.py @@ -0,0 +1 @@ +from .ftx_exchange import FTX \ No newline at end of file diff --git a/Trading/Exchange/ftx/ftx_exchange.py b/Trading/Exchange/ftx/ftx_exchange.py new file mode 100644 index 000000000..97445e4df --- /dev/null +++ b/Trading/Exchange/ftx/ftx_exchange.py @@ -0,0 +1,70 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import copy +import math + +import ccxt + +import octobot_trading.exchanges as exchanges +import octobot_trading.errors +import octobot_trading.enums as trading_enums + + +class FTX(exchanges.SpotCCXTExchange): + DESCRIPTION = "" + + @classmethod + def get_name(cls): + return 'ftx' + + @classmethod + def is_supporting_exchange(cls, exchange_candidate_name) -> bool: + return cls.get_name() == exchange_candidate_name + + def get_market_status(self, symbol, price_example=None, with_fixer=True): + try: + market_status = self._fix_market_status(copy.deepcopy(self.connector.client.market(symbol))) + if with_fixer: + market_status = exchanges.ExchangeMarketStatusFixer(market_status, price_example).market_status + return market_status + except ccxt.NotSupported: + raise octobot_trading.errors.NotSupported + except Exception as e: + self.logger.error(f"Fail to get market status of {symbol}: {e}") + return {} + + def _fix_market_status(self, market_status): + market_status[trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION.value][ + trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION_AMOUNT.value] = self._get_digits_count( + market_status[trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION.value][ + trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION_AMOUNT.value]) + market_status[trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION.value][ + trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION_PRICE.value] = self._get_digits_count( + market_status[trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION.value][ + trading_enums.ExchangeConstantsMarketStatusColumns.PRECISION_PRICE.value]) + return market_status + + def _get_digits_count(self, value): + return round(abs(math.log(value, 10))) + + async def get_price_ticker(self, symbol: str, **kwargs: dict): + try: + ticker = await self.connector.client.fetch_ticker(symbol, params=kwargs) + ticker[trading_enums.ExchangeConstantsTickersColumns.TIMESTAMP.value] = self.connector.client.milliseconds() + ticker[trading_enums.ExchangeConstantsTickersColumns.BASE_VOLUME.value] = ticker[trading_enums.ExchangeConstantsTickersColumns.QUOTE_VOLUME.value] / ticker[trading_enums.ExchangeConstantsTickersColumns.CLOSE.value] + return ticker + except ccxt.BaseError as e: + raise octobot_trading.errors.FailedRequest(f"Failed to get_price_ticker {e}") diff --git a/Trading/Exchange/ftx/metadata.json b/Trading/Exchange/ftx/metadata.json new file mode 100644 index 000000000..f024634ee --- /dev/null +++ b/Trading/Exchange/ftx/metadata.json @@ -0,0 +1,6 @@ +{ + "version": "1.2.0", + "origin_package": "OctoBot-Default-Tentacles", + "tentacles": ["FTX"], + "tentacles-requirements": [] +} \ No newline at end of file diff --git a/Trading/Exchange/ftx/resources/ftx.md b/Trading/Exchange/ftx/resources/ftx.md new file mode 100644 index 000000000..21eb98161 --- /dev/null +++ b/Trading/Exchange/ftx/resources/ftx.md @@ -0,0 +1 @@ +FTX is a basic SpotExchange adaptation for FTX exchange. diff --git a/Trading/Exchange/ftx/tests/__init__.py b/Trading/Exchange/ftx/tests/__init__.py new file mode 100644 index 000000000..974dd1623 --- /dev/null +++ b/Trading/Exchange/ftx/tests/__init__.py @@ -0,0 +1,15 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. From 8e8222a8c2566d13bc4ed45dcb5af2134a9129cc Mon Sep 17 00:00:00 2001 From: herklos Date: Tue, 8 Jun 2021 23:07:28 +0200 Subject: [PATCH 02/44] [Trading][Exchanges] Add wavesexchange --- Trading/Exchange/wavesexchange/__init__.py | 1 + Trading/Exchange/wavesexchange/metadata.json | 6 +++ .../wavesexchange/resources/wavesexchange.md | 1 + .../Exchange/wavesexchange/tests/__init__.py | 15 +++++++ .../wavesexchange/wavesexchange_exchange.py | 40 +++++++++++++++++++ 5 files changed, 63 insertions(+) create mode 100644 Trading/Exchange/wavesexchange/__init__.py create mode 100644 Trading/Exchange/wavesexchange/metadata.json create mode 100644 Trading/Exchange/wavesexchange/resources/wavesexchange.md create mode 100644 Trading/Exchange/wavesexchange/tests/__init__.py create mode 100644 Trading/Exchange/wavesexchange/wavesexchange_exchange.py diff --git a/Trading/Exchange/wavesexchange/__init__.py b/Trading/Exchange/wavesexchange/__init__.py new file mode 100644 index 000000000..40ca1a675 --- /dev/null +++ b/Trading/Exchange/wavesexchange/__init__.py @@ -0,0 +1 @@ +from .wavesexchange_exchange import WavesExchange \ No newline at end of file diff --git a/Trading/Exchange/wavesexchange/metadata.json b/Trading/Exchange/wavesexchange/metadata.json new file mode 100644 index 000000000..d7444a5e0 --- /dev/null +++ b/Trading/Exchange/wavesexchange/metadata.json @@ -0,0 +1,6 @@ +{ + "version": "1.2.0", + "origin_package": "OctoBot-Default-Tentacles", + "tentacles": ["WavesExchange"], + "tentacles-requirements": [] +} \ No newline at end of file diff --git a/Trading/Exchange/wavesexchange/resources/wavesexchange.md b/Trading/Exchange/wavesexchange/resources/wavesexchange.md new file mode 100644 index 000000000..4a0cafb95 --- /dev/null +++ b/Trading/Exchange/wavesexchange/resources/wavesexchange.md @@ -0,0 +1 @@ +WavesExchange is a basic SpotExchange adaptation for WavesExchange exchange. diff --git a/Trading/Exchange/wavesexchange/tests/__init__.py b/Trading/Exchange/wavesexchange/tests/__init__.py new file mode 100644 index 000000000..974dd1623 --- /dev/null +++ b/Trading/Exchange/wavesexchange/tests/__init__.py @@ -0,0 +1,15 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. diff --git a/Trading/Exchange/wavesexchange/wavesexchange_exchange.py b/Trading/Exchange/wavesexchange/wavesexchange_exchange.py new file mode 100644 index 000000000..680884215 --- /dev/null +++ b/Trading/Exchange/wavesexchange/wavesexchange_exchange.py @@ -0,0 +1,40 @@ +# Drakkar-Software OctoBot-Tentacles +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import ccxt + +import octobot_trading.exchanges as exchanges +import octobot_trading.errors +import octobot_trading.enums as trading_enums + + +class WavesExchange(exchanges.SpotCCXTExchange): + DESCRIPTION = "" + + @classmethod + def get_name(cls): + return 'wavesexchange' + + @classmethod + def is_supporting_exchange(cls, exchange_candidate_name) -> bool: + return cls.get_name() == exchange_candidate_name + + async def get_price_ticker(self, symbol: str, **kwargs: dict): + try: + ticker = await self.connector.client.fetch_ticker(symbol, params=kwargs) + ticker[trading_enums.ExchangeConstantsTickersColumns.TIMESTAMP.value] = self.connector.client.milliseconds() + return ticker + except ccxt.BaseError as e: + raise octobot_trading.errors.FailedRequest(f"Failed to get_price_ticker {e}") From 02ad882ea67e0e4363ccb4eefc6e35ed0c0b7116 Mon Sep 17 00:00:00 2001 From: Herklos Date: Tue, 15 Jun 2021 21:36:09 +0200 Subject: [PATCH 03/44] [BinanceExchange] Remove unused constant --- Trading/Exchange/binance/binance_exchange.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Trading/Exchange/binance/binance_exchange.py b/Trading/Exchange/binance/binance_exchange.py index b3c43c72a..980e35d35 100644 --- a/Trading/Exchange/binance/binance_exchange.py +++ b/Trading/Exchange/binance/binance_exchange.py @@ -29,8 +29,6 @@ class Binance(exchanges.SpotCCXTExchange): BINANCE_MARK_PRICE = "markPrice" - CCXT_CLIENT_LOGIN_OPTIONS = {'defaultMarket': 'future'} - @classmethod def get_name(cls): return 'binance' From 302cf2f601fbeb42913405635c788c19fc7a657c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Coss=C3=A9?= Date: Mon, 21 Jun 2021 12:33:09 +0200 Subject: [PATCH 04/44] [Data-Collector] Reset symbol select index after update --- .../web_interface/static/js/components/data_collector.js | 1 + 1 file changed, 1 insertion(+) diff --git a/Services/Interfaces/web_interface/static/js/components/data_collector.js b/Services/Interfaces/web_interface/static/js/components/data_collector.js index ee9521401..b5e00cad1 100644 --- a/Services/Interfaces/web_interface/static/js/components/data_collector.js +++ b/Services/Interfaces/web_interface/static/js/components/data_collector.js @@ -101,6 +101,7 @@ function update_symbol_list(url, exchange){ symbolSelect.append($("") .attr("value", value).text(value)); }); + symbolSelect[0].selectedIndex = -1; symbolSelect.selectpicker('refresh'); }); } From 9648110ad3495283658cdb8830e6da5597f5393a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Coss=C3=A9?= Date: Sat, 19 Jun 2021 16:56:15 +0200 Subject: [PATCH 05/44] [WebInterface] Check tentacles config on save --- .../static/js/components/config_tentacle.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Services/Interfaces/web_interface/static/js/components/config_tentacle.js b/Services/Interfaces/web_interface/static/js/components/config_tentacle.js index d5b075c23..a0153bfb9 100644 --- a/Services/Interfaces/web_interface/static/js/components/config_tentacle.js +++ b/Services/Interfaces/web_interface/static/js/components/config_tentacle.js @@ -61,12 +61,25 @@ function handle_tentacle_config_update_error_callback(updated_data, update_url, create_alert("error", "Error when updating config", msg.responseText); } +function check_config(){ + var errors = configEditor.validate(); + + if(errors.length) + return false + else + return true + +} + function handleConfigDisplay(){ if(canEditConfig()){ $("#saveConfigFooter").show(); $("#saveConfig").click(function() { - updateTentacleConfig(configEditor.getValue()); + if(check_config()) + updateTentacleConfig(configEditor.getValue()); + else + alert("Invalid data form.") }); }else{ $("#noConfigMessage").show(); From 32fa3814f0dab572e6c104e57f4fae273b4aeeb7 Mon Sep 17 00:00:00 2001 From: valouvaliavlo Date: Sun, 20 Jun 2021 22:28:26 +0200 Subject: [PATCH 06/44] Apply suggestions from code review Co-authored-by: Guillaume De Saint Martin --- .../web_interface/static/js/components/config_tentacle.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Services/Interfaces/web_interface/static/js/components/config_tentacle.js b/Services/Interfaces/web_interface/static/js/components/config_tentacle.js index a0153bfb9..3aa99e0c8 100644 --- a/Services/Interfaces/web_interface/static/js/components/config_tentacle.js +++ b/Services/Interfaces/web_interface/static/js/components/config_tentacle.js @@ -67,7 +67,8 @@ function check_config(){ if(errors.length) return false else - return true + const errors = configEditor.validate(); + return !errors.length; } @@ -79,7 +80,7 @@ function handleConfigDisplay(){ if(check_config()) updateTentacleConfig(configEditor.getValue()); else - alert("Invalid data form.") + create_alert("error", "Error when saving configuration", "Invalid configuration data."); }); }else{ $("#noConfigMessage").show(); From fedc7c3fbd5e56c70fd931b6c0b60e12b70ad7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Coss=C3=A9?= Date: Mon, 21 Jun 2021 11:56:17 +0200 Subject: [PATCH 07/44] Apply suggestion from code review --- .../web_interface/static/js/components/config_tentacle.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Services/Interfaces/web_interface/static/js/components/config_tentacle.js b/Services/Interfaces/web_interface/static/js/components/config_tentacle.js index 3aa99e0c8..4603e7b0d 100644 --- a/Services/Interfaces/web_interface/static/js/components/config_tentacle.js +++ b/Services/Interfaces/web_interface/static/js/components/config_tentacle.js @@ -62,14 +62,8 @@ function handle_tentacle_config_update_error_callback(updated_data, update_url, } function check_config(){ - var errors = configEditor.validate(); - - if(errors.length) - return false - else const errors = configEditor.validate(); return !errors.length; - } function handleConfigDisplay(){ From ba715eb4da600527ef9a13e980883bbebb1a748f Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Sun, 27 Jun 2021 22:24:15 +0200 Subject: [PATCH 08/44] [WebInterface] fix errors on backtesting files diplay and currency fetch --- Services/Interfaces/web_interface/models/backtesting.py | 8 +++++++- Services/Interfaces/web_interface/models/configuration.py | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Services/Interfaces/web_interface/models/backtesting.py b/Services/Interfaces/web_interface/models/backtesting.py index 547c02cbe..412edc8a8 100644 --- a/Services/Interfaces/web_interface/models/backtesting.py +++ b/Services/Interfaces/web_interface/models/backtesting.py @@ -30,10 +30,16 @@ async def _get_description(data_file, files_with_description): description = await backtesting_api.get_file_description(data_file) - if description is not None: + if _is_usable_description(description): files_with_description[data_file] = description +def _is_usable_description(description): + return description is not None \ + and description[backtesting_enums.DataFormatKeys.SYMBOLS.value] is not None \ + and description[backtesting_enums.DataFormatKeys.TIME_FRAMES.value] is not None + + async def _retrieve_data_files_with_description(files): files_with_description = {} await asyncio.gather(*[_get_description(data_file, files_with_description) for data_file in files]) diff --git a/Services/Interfaces/web_interface/models/configuration.py b/Services/Interfaces/web_interface/models/configuration.py index cc8ffb137..4e6db3957 100644 --- a/Services/Interfaces/web_interface/models/configuration.py +++ b/Services/Interfaces/web_interface/models/configuration.py @@ -569,9 +569,10 @@ def get_all_symbols_dict(): if _is_legit_currency(currency_data[NAME_KEY]) } except Exception as e: + details = f"code: {request_response.status_code}, body: {request_response.text}" \ + if request_response else {request_response} _get_logger().error(f"Failed to get currencies list from coingecko.com : {e}") - _get_logger().debug(f"coingecko.com response code: {request_response.status_code}, body: " - f"{request_response.text}") + _get_logger().debug(f"coingecko.com response {details}") return {} return all_symbols_dict From 95d4554755447cc6cce007145005227b1ad86f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Valentin=20Coss=C3=A9?= Date: Mon, 21 Jun 2021 00:57:59 +0200 Subject: [PATCH 09/44] [Data-Collector] Get history on date range --- .../history_collector.py | 27 ++++++- .../web_interface/controllers/backtesting.py | 8 ++- .../web_interface/models/backtesting.py | 19 ++++- .../static/js/components/data_collector.js | 12 +++- .../templates/data_collector.html | 72 ++++++++++++------- 5 files changed, 104 insertions(+), 34 deletions(-) diff --git a/Backtesting/collectors/exchanges/exchange_history_collector/history_collector.py b/Backtesting/collectors/exchanges/exchange_history_collector/history_collector.py index 4bb455817..fe6b53c95 100644 --- a/Backtesting/collectors/exchanges/exchange_history_collector/history_collector.py +++ b/Backtesting/collectors/exchanges/exchange_history_collector/history_collector.py @@ -34,11 +34,15 @@ class ExchangeHistoryDataCollector(collector.AbstractExchangeHistoryCollector): def __init__(self, config, exchange_name, tentacles_setup_config, symbols, time_frames, use_all_available_timeframes=False, - data_format=backtesting_enums.DataFormats.REGULAR_COLLECTOR_DATA): + data_format=backtesting_enums.DataFormats.REGULAR_COLLECTOR_DATA, + start_timestamp=None, + end_timestamp=None): super().__init__(config, exchange_name, tentacles_setup_config, symbols, time_frames, use_all_available_timeframes, data_format=data_format) self.exchange = None self.exchange_manager = None + self.start_timestamp = start_timestamp + self.end_timestamp = end_timestamp async def start(self): should_stop_database = True @@ -106,12 +110,31 @@ async def get_recent_trades_history(self, exchange, symbol): async def get_ohlcv_history(self, exchange, symbol, time_frame): # use time_frame_sec to add time to save the candle closing time time_frame_sec = commons_enums.TimeFramesMinutes[time_frame] * commons_constants.MINUTE_TO_SECONDS - candles = await self.exchange.get_symbol_prices(symbol, time_frame) + + if self.start_timestamp is not None or self.end_timestamp is not None: + since = self.start_timestamp + if self.start_timestamp is None or self.start_timestamp < await self.get_first_candle_timestamp(symbol, time_frame): + since=0 + candles = await self.exchange.get_symbol_prices(symbol, time_frame, since=since) + while since < candles[-1][0] if not self.end_timestamp \ + else (candles[-1][0] < self.end_timestamp): + since = candles[-1][0] + candles += await self.exchange.get_symbol_prices(symbol, time_frame, since=since) + if self.end_timestamp is not None: + while candles[-1][0] > self.end_timestamp: + candles.pop(-1) + else: + candles = await self.exchange.get_symbol_prices(symbol, time_frame) + self.exchange.uniformize_candles_if_necessary(candles) await self.save_ohlcv(exchange=exchange, cryptocurrency=self.exchange_manager.exchange.get_pair_cryptocurrency(symbol), symbol=symbol, time_frame=time_frame, candle=candles, timestamp=[candle[0] + time_frame_sec for candle in candles], multiple=True) + async def get_kline_history(self, exchange, symbol, time_frame): pass + + async def get_first_candle_timestamp(self, symbol, time_frame): + return (await self.exchange.get_symbol_prices(symbol, time_frame, since=0))[-1][0] diff --git a/Services/Interfaces/web_interface/controllers/backtesting.py b/Services/Interfaces/web_interface/controllers/backtesting.py index 7b836367f..e739d14e6 100644 --- a/Services/Interfaces/web_interface/controllers/backtesting.py +++ b/Services/Interfaces/web_interface/controllers/backtesting.py @@ -73,7 +73,7 @@ def data_collector(): success, reply = models.get_delete_data_file(file) elif action_type == "start_collector": details = flask.request.get_json() - success, reply = models.collect_data_file(details["exchange"], details["symbol"]) + success, reply = models.collect_data_file(details["exchange"], details["symbol"], details["startTimestamp"], details["endTimestamp"]) elif action_type == "import_data_file": if flask.request.files: file = flask.request.files['file'] @@ -87,7 +87,8 @@ def data_collector(): # here return template to force page reload because of file upload via input form return flask.render_template('data_collector.html', data_files=models.get_data_files_with_description(), - ccxt_exchanges=sorted(models.get_full_exchange_list()), + other_ccxt_exchanges=sorted(models.get_other_history_exchange_list()), + full_history_ccxt_exchanges=models.get_full_history_exchange_list(), current_exchange=models.get_current_exchange(), full_symbol_list=sorted(models.get_symbol_list([current_exchange])), alert=alert) @@ -112,7 +113,8 @@ def data_collector(): current_exchange = models.get_current_exchange() return flask.render_template('data_collector.html', data_files=models.get_data_files_with_description(), - ccxt_exchanges=sorted(models.get_full_exchange_list()), + other_ccxt_exchanges=sorted(models.get_other_history_exchange_list()), + full_history_ccxt_exchanges=models.get_full_history_exchange_list(), current_exchange=models.get_current_exchange(), full_symbol_list=sorted(models.get_symbol_list([current_exchange])), origin_page=origin_page, diff --git a/Services/Interfaces/web_interface/models/backtesting.py b/Services/Interfaces/web_interface/models/backtesting.py index 412edc8a8..c3da81415 100644 --- a/Services/Interfaces/web_interface/models/backtesting.py +++ b/Services/Interfaces/web_interface/models/backtesting.py @@ -15,6 +15,7 @@ # License along with this library. import os import asyncio +import ccxt import octobot_commons.logging as bot_logging import octobot.api as octobot_api @@ -22,12 +23,24 @@ import octobot_tentacles_manager.api as tentacles_manager_api import octobot_backtesting.constants as backtesting_constants import octobot_services.interfaces.util as interfaces_util +import octobot_trading.constants as trading_constants import tentacles.Services.Interfaces.web_interface.constants as constants import tentacles.Services.Interfaces.web_interface as web_interface_root + LOGGER = bot_logging.get_logger("DataCollectorWebInterfaceModel") +def get_full_history_exchange_list(): + full_exchange_list = list(set(ccxt.exchanges)) + return [exchange for exchange in trading_constants.FULL_HISTORY_EXCHANGES if exchange in full_exchange_list] + + +def get_other_history_exchange_list(): + full_exchange_list = list(set(ccxt.exchanges)) + return [exchange for exchange in full_exchange_list if exchange not in trading_constants.FULL_HISTORY_EXCHANGES] + + async def _get_description(data_file, files_with_description): description = await backtesting_api.get_file_description(data_file) if _is_usable_description(description): @@ -115,13 +128,15 @@ def get_delete_data_file(file_name): return deleted, f"Can't delete {file_name} ({error})" -def collect_data_file(exchange, symbol): +def collect_data_file(exchange, symbol, start_timestamp=None, end_timestamp=None): success = False try: result = interfaces_util.run_in_bot_async_executor( backtesting_api.collect_exchange_historical_data(exchange, interfaces_util.get_bot_api().get_edited_tentacles_config(), - [symbol])) + [symbol], + start_timestamp=start_timestamp, + end_timestamp=end_timestamp)) success = True except Exception as e: result = f"data collector error: {e}" diff --git a/Services/Interfaces/web_interface/static/js/components/data_collector.js b/Services/Interfaces/web_interface/static/js/components/data_collector.js index b5e00cad1..ed4b57375 100644 --- a/Services/Interfaces/web_interface/static/js/components/data_collector.js +++ b/Services/Interfaces/web_interface/static/js/components/data_collector.js @@ -69,6 +69,8 @@ function start_collector(){ const request = {}; request["exchange"] = $('#exchangeSelect').val(); request["symbol"] = $('#symbolSelect').val(); + request["startTimestamp"] = is_full_history_exchanges() ? (new Date($("#startDate").val()).getTime()) : float("NaN"); + request["endTimestamp"] = is_full_history_exchanges() ? (new Date($("#endDate").val()).getTime()) : float("NaN"); const update_url = $("#collect_data").attr(update_url_attr); send_and_interpret_bot_update(request, update_url, $(this), collector_success_callback, collector_error_callback) } @@ -106,16 +108,24 @@ function update_symbol_list(url, exchange){ }); } +function is_full_history_exchanges(){ + const full_history_exchanges = $('#exchangeSelect > optgroup')[0].children; + const selected_exchange = $('#exchangeSelect').find(":selected")[0]; + return $.inArray(selected_exchange, full_history_exchanges) !== -1 +} + let dataFilesTable = $('#dataFilesTable').DataTable({"order": []}); $(document).ready(function() { handle_data_files_buttons(); + is_full_history_exchanges() ? $("#collector_date_range").show() : $("#collector_date_range").hide(); $('#importFileButton').attr('disabled', true); dataFilesTable.on("draw.dt", function(){ handle_data_files_buttons(); }); $('#exchangeSelect').on('change', function() { - update_symbol_list($('#symbolSelect').attr(update_url_attr), $('#exchangeSelect').val()) + update_symbol_list($('#symbolSelect').attr(update_url_attr), $('#exchangeSelect').val()); + is_full_history_exchanges() ? $("#collector_date_range").show() : $("#collector_date_range").hide(); }); $('#collect_data').click(function(){ start_collector(); diff --git a/Services/Interfaces/web_interface/templates/data_collector.html b/Services/Interfaces/web_interface/templates/data_collector.html index 1d6bcb5f6..7c2e93ae9 100644 --- a/Services/Interfaces/web_interface/templates/data_collector.html +++ b/Services/Interfaces/web_interface/templates/data_collector.html @@ -47,32 +47,52 @@

Download exchange data

-
- Exchange : - -
-
- Symbol : - -
-
- -
+
+ Exchange : + +
+
+ Symbol : + +
+
+ +
+
+
+
+ Start Date : + +
+
+ End Date : + +