Merge pull request #500 from Drakkar-Software/dev

master merge

.github/workflows/main.yml

@@ -32,7 +32,8 @@ jobs:
         OCTOBOT_DEFAULT_BRANCH: dev
         DISABLE_SENTRY: True
       run: |
-        git clone -q $OCTOBOT_GH_REPO -b ${GITHUB_REF##*/} || git clone -q $OCTOBOT_GH_REPO -b $OCTOBOT_DEFAULT_BRANCH
+        TARGET_BRANCH=$([ "$GITHUB_HEAD_REF" == "" ] && echo ${GITHUB_REF##*/} || echo "$GITHUB_HEAD_REF")
+        git clone -q $OCTOBOT_GH_REPO -b ${TARGET_BRANCH} || git clone -q $OCTOBOT_GH_REPO -b $OCTOBOT_DEFAULT_BRANCH
         cd OctoBot
         git status
         pip install --prefer-binary -r dev_requirements.txt -r requirements.txt
@@ -50,10 +51,11 @@ jobs:
         OCTOBOT_DEFAULT_BRANCH: dev
         DISABLE_SENTRY: True
       run: |
-        If ($env:GITHUB_REF -notcontains "refs/tags/") {
+        $env:TARGET_BRANCH = @({$env:GITHUB_HEAD_REF}, {$env:GITHUB_REF})[(Test-Path env:GITHUB_HEAD_REF)]
+        If ($env:TARGET_BRANCH -notcontains "refs/tags/") {
           $env:TENTACLES_URL_TAG = "latest"
         }
-        git clone -q $env:OCTOBOT_GH_REPO -b $env:GITHUB_REF.Replace('refs/heads/','')
+        git clone -q $env:OCTOBOT_GH_REPO -b $env:TARGET_BRANCH.Replace('refs/heads/','')
         if ($LastExitCode -ne 0) {
           git clone -q $env:OCTOBOT_GH_REPO -b $env:OCTOBOT_DEFAULT_BRANCH
         }
@@ -115,6 +117,8 @@ jobs:
           echo "NEXUS_USERNAME=${{ secrets.NEXUS_USERNAME }}" >> $GITHUB_ENV
           echo "NEXUS_PASSWORD=${{ secrets.NEXUS_PASSWORD }}" >> $GITHUB_ENV
           echo "NEXUS_URL=${{ secrets.NEXUS_URL }}" >> $GITHUB_ENV
+          TARGET_BRANCH=$([ "$GITHUB_HEAD_REF" == "" ] && echo ${GITHUB_REF##*/} || echo "$GITHUB_HEAD_REF")
+          echo "TARGET_BRANCH=${TARGET_BRANCH}" >> $GITHUB_ENV
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
@@ -127,7 +131,7 @@ jobs:
           OCTOBOT_GH_REPO: https://github.com/Drakkar-Software/OctoBot.git
           OCTOBOT_DEFAULT_BRANCH: dev
         run: |
-          git clone -q $OCTOBOT_GH_REPO -b ${GITHUB_REF##*/} || git clone -q $OCTOBOT_GH_REPO -b $OCTOBOT_DEFAULT_BRANCH
+          git clone -q $OCTOBOT_GH_REPO -b ${TARGET_BRANCH} || git clone -q $OCTOBOT_GH_REPO -b $OCTOBOT_DEFAULT_BRANCH
           cd OctoBot
           git status
           pip install --prefer-binary -r dev_requirements.txt -r requirements.txt
@@ -138,7 +142,7 @@ jobs:
       - name: Publish tag tentacles
         if: startsWith(github.ref, 'refs/tags')
         run: |
-          sed -i "s/VERSION_PLACEHOLDER/${GITHUB_REF#refs/*/}/g" metadata.yaml
+          sed -i "s/VERSION_PLACEHOLDER/${TARGET_BRANCH#refs/*/}/g" metadata.yaml
           cd OctoBot
           python start.py tentacles -m "../metadata.yaml" -d "../new_tentacles" -p "../../any_platform.zip" -ite -ute ${{ secrets.NEXUS_OFFICIAL_PATH }}/tentacles -upe ${{ secrets.NEXUS_OFFICIAL_PATH }}/packages/full/${{ secrets.TENTACLES_REPOSITORY_NAME }}
 
@@ -154,7 +158,7 @@ jobs:
         env:
           NEXUS_URL: ${{ secrets.NEXUS_DEV_URL }}
         run: |
-          branch="${GITHUB_REF##*/}"
+          branch="${TARGET_BRANCH##*/}"
           sed -i "s/VERSION_PLACEHOLDER/$branch/g" metadata.yaml
           sed -i "s/base/$branch/g" metadata.yaml
           sed -i "s/officials/dev/g" metadata.yaml

Backtesting/collectors/exchanges/exchange_history_collector/history_collector.py

@@ -15,12 +15,14 @@
 #  License along with this library.
 import logging
 import os
+import time
 
 import octobot_backtesting.collectors as collector
 import octobot_backtesting.enums as backtesting_enums
 import octobot_backtesting.errors as errors
 import octobot_commons.constants as commons_constants
 import octobot_commons.enums as commons_enums
+import octobot_commons.time_frame_manager as time_frame_manager
 import tentacles.Backtesting.importers.exchanges.generic_exchange_importer as generic_exchange_importer
 
 try:
@@ -34,9 +36,12 @@ class ExchangeHistoryDataCollector(collector.AbstractExchangeHistoryCollector):
 
     def __init__(self, config, exchange_name, tentacles_setup_config, symbols, time_frames,
                  use_all_available_timeframes=False,
-                 data_format=backtesting_enums.DataFormats.REGULAR_COLLECTOR_DATA):
+                 data_format=backtesting_enums.DataFormats.REGULAR_COLLECTOR_DATA,
+                 start_timestamp=None,
+                 end_timestamp=None):
         super().__init__(config, exchange_name, tentacles_setup_config, symbols, time_frames,
-                         use_all_available_timeframes, data_format=data_format)
+                         use_all_available_timeframes, data_format=data_format,
+                         start_timestamp=start_timestamp, end_timestamp=end_timestamp)
         self.exchange = None
         self.exchange_manager = None
 
@@ -56,6 +61,15 @@ async def start(self):
             self.exchange = self.exchange_manager.exchange
             self._load_timeframes_if_necessary()
 
+            if self.start_timestamp is not None:
+                lowest_timestamp = min([await self.get_first_candle_timestamp(symbol,
+                                        time_frame_manager.find_min_time_frame(self.time_frames))
+                                        for symbol in self.symbols])
+                if lowest_timestamp > self.start_timestamp:
+                    self.start_timestamp = lowest_timestamp
+                if self.start_timestamp > (self.end_timestamp if self.end_timestamp else (time.time()*1000)):
+                    raise errors.DataCollectorError("start_timestamp is higher than end_timestamp")
+
             # create description
             await self._create_description()
 
@@ -66,8 +80,9 @@ async def start(self):
                 await self.get_order_book_history(self.exchange_name, symbol)
                 await self.get_recent_trades_history(self.exchange_name, symbol)
 
-                for time_frame in self.time_frames:
-                    self.logger.info(f"Collecting history on {time_frame}...")
+                for index, time_frame in enumerate(self.time_frames):
+                    self.logger.info(
+                        f"[{index}/{len(self.time_frames)}] Collecting {symbol} history on {time_frame}...")
                     await self.get_ohlcv_history(self.exchange_name, symbol, time_frame)
                     await self.get_kline_history(self.exchange_name, symbol, time_frame)
         except Exception as err:
@@ -76,8 +91,8 @@ async def start(self):
             await self.database.stop()
             should_stop_database = False
             # Do not keep errored data file
-            if os.path.isfile(self.file_path):
-                os.remove(self.file_path)
+            if os.path.isfile(self.temp_file_path):
+                os.remove(self.temp_file_path)
             raise errors.DataCollectorError(err)
         finally:
             await self.stop(should_stop_database=should_stop_database)
@@ -92,6 +107,7 @@ async def stop(self, should_stop_database=True):
         await self.exchange_manager.stop()
         if should_stop_database:
             await self.database.stop()
+            self.finalize_database()
         self.exchange_manager = None
 
     async def get_ticker_history(self, exchange, symbol):
@@ -106,7 +122,37 @@ async def get_recent_trades_history(self, exchange, symbol):
     async def get_ohlcv_history(self, exchange, symbol, time_frame):
         # use time_frame_sec to add time to save the candle closing time
         time_frame_sec = commons_enums.TimeFramesMinutes[time_frame] * commons_constants.MINUTE_TO_SECONDS
-        candles = await self.exchange.get_symbol_prices(symbol, time_frame)
+
+        if self.start_timestamp is not None:
+            first_candle_timestamp = await self.get_first_candle_timestamp(symbol, time_frame)
+            since = self.start_timestamp
+
+            if self.start_timestamp < first_candle_timestamp:
+                since = first_candle_timestamp
+
+            if ((self.end_timestamp or time.time()*1000) - since) < (time_frame_sec * 1000):
+                return
+
+            candles = await self.exchange.get_symbol_prices(symbol, time_frame, since=since)
+            last_candle_timestamp = candles[-1][commons_enums.PriceIndexes.IND_PRICE_TIME.value]
+
+            total_interval = (self.end_timestamp or (time.time()*1000)) - last_candle_timestamp
+            start_fetch_time = last_candle_timestamp
+            while since < last_candle_timestamp if not self.end_timestamp \
+                    else (last_candle_timestamp < self.end_timestamp - (time_frame_sec * 1000)):
+                since = last_candle_timestamp
+                progress = (since-start_fetch_time) / total_interval
+                self.logger.info(f"[{round(progress *100)}%] historical data fetched for {symbol} {time_frame}")
+                candles += await self.exchange.get_symbol_prices(symbol, time_frame,
+                                                                 since=(since + (time_frame_sec * 1000)))
+                last_candle_timestamp = candles[-1][commons_enums.PriceIndexes.IND_PRICE_TIME.value]
+
+            if self.end_timestamp is not None:
+                while candles[-1][commons_enums.PriceIndexes.IND_PRICE_TIME.value] > self.end_timestamp:
+                    candles.pop(-1)
+        else:
+            candles = await self.exchange.get_symbol_prices(symbol, time_frame)
+
         self.exchange.uniformize_candles_if_necessary(candles)
         await self.save_ohlcv(exchange=exchange,
                               cryptocurrency=self.exchange_manager.exchange.get_pair_cryptocurrency(symbol),
@@ -115,3 +161,6 @@ async def get_ohlcv_history(self, exchange, symbol, time_frame):
 
     async def get_kline_history(self, exchange, symbol, time_frame):
         pass
+
+    async def get_first_candle_timestamp(self, symbol, time_frame):
+        return (await self.exchange.get_symbol_prices(symbol, time_frame, limit=1, since=0))[0][commons_enums.PriceIndexes.IND_PRICE_TIME.value]

Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py

@@ -29,17 +29,21 @@
 
 
 @contextlib.asynccontextmanager
-async def data_collector(exchange_name, tentacles_setup_config, symbols, time_frames, use_all_available_timeframes):
+async def data_collector(exchange_name, tentacles_setup_config, symbols, time_frames, use_all_available_timeframes, start_timestamp=None, end_timestamp=None):
     collector_instance = collector_exchanges.ExchangeHistoryDataCollector(
         {}, exchange_name, tentacles_setup_config, symbols, time_frames,
-        use_all_available_timeframes=use_all_available_timeframes
+        use_all_available_timeframes=use_all_available_timeframes,
+        start_timestamp=start_timestamp,
+        end_timestamp=end_timestamp
     )
     try:
         await collector_instance.initialize()
         yield collector_instance
     finally:
         if collector_instance.file_path and os.path.isfile(collector_instance.file_path):
             os.remove(collector_instance.file_path)
+        if collector_instance.temp_file_path and os.path.isfile(collector_instance.temp_file_path):
+            os.remove(collector_instance.temp_file_path)
 
 
 @contextlib.asynccontextmanager
@@ -66,6 +70,9 @@ async def test_collect_valid_data():
         assert collector.exchange_manager is None
         assert isinstance(collector.exchange, tentacles_exchanges.Binance)
         assert collector.file_path is not None
+        assert collector.temp_file_path is not None
+        assert not os.path.isfile(collector.temp_file_path)
+        assert os.path.isfile(collector.file_path)
         async with collector_database(collector) as database:
             ohlcv = await database.select(enums.ExchangeDataTables.OHLCV)
             assert len(ohlcv) > 6000
@@ -86,4 +93,57 @@ async def test_collect_invalid_data():
         assert collector.exchange_manager is None
         assert collector.exchange is not None
         assert collector.file_path is not None
+        assert collector.temp_file_path is not None
+        assert not os.path.isfile(collector.temp_file_path)
+
+async def test_collect_valid_date_range():
+    exchange_name = "binance"
+    tentacles_setup_config = test_utils_config.load_test_tentacles_config()
+    symbols = ["ETH/BTC"]
+    async with data_collector(exchange_name, tentacles_setup_config, symbols, None, True, 1549065660000, 1549670520000) as collector:
+        assert collector.time_frames == []
+        assert collector.symbols == symbols
+        assert collector.exchange_name == exchange_name
+        assert collector.tentacles_setup_config == tentacles_setup_config
+        assert collector.start_timestamp is not None
+        assert collector.end_timestamp is not None
+        await collector.start()
+        assert collector.time_frames != []
+        assert collector.exchange_manager is None
+        assert isinstance(collector.exchange, tentacles_exchanges.Binance)
+        assert collector.file_path is not None
+        assert collector.temp_file_path is not None
+        assert os.path.isfile(collector.file_path)
+        assert not os.path.isfile(collector.temp_file_path)
+        async with collector_database(collector) as database:
+            ohlcv = await database.select(enums.ExchangeDataTables.OHLCV)
+            assert len(ohlcv) == 16833
+            h_ohlcv = await database.select(enums.ExchangeDataTables.OHLCV, time_frame="1h")
+            assert len(h_ohlcv) == 168
+            eth_btc_ohlcv = await database.select(enums.ExchangeDataTables.OHLCV, symbol="ETH/BTC")
+            assert len(eth_btc_ohlcv) == len(ohlcv)
+            min_timestamp = (await database.select_min(enums.ExchangeDataTables.OHLCV, ["timestamp"],time_frame="1m"))[0][0]*1000
+            assert min_timestamp <= 1549065720000
+            max_timestamp = (await database.select_max(enums.ExchangeDataTables.OHLCV, ["timestamp"]))[0][0]*1000
+            assert max_timestamp <= 1549843200000
+
+async def test_collect_invalid_date_range():
+    exchange_name = "binance"
+    tentacles_setup_config = test_utils_config.load_test_tentacles_config()
+    symbols = ["ETH/BTC"]
+    async with data_collector(exchange_name, tentacles_setup_config, symbols, None, True, 1609459200, 1577836800) as collector:
+        assert collector.time_frames == []
+        assert collector.symbols == symbols
+        assert collector.exchange_name == exchange_name
+        assert collector.tentacles_setup_config == tentacles_setup_config
+        assert collector.start_timestamp is not None
+        assert collector.end_timestamp is not None
+        with pytest.raises(errors.DataCollectorError):
+            await collector.start()
+        assert collector.time_frames != []
+        assert collector.exchange_manager is None
+        assert isinstance(collector.exchange, tentacles_exchanges.Binance)
+        assert collector.file_path is not None
+        assert collector.temp_file_path is not None
         assert not os.path.isfile(collector.file_path)
+        assert not os.path.isfile(collector.temp_file_path)

Services/Interfaces/web_interface/api/__init__.py

@@ -18,11 +18,15 @@
 
 api = flask.Blueprint('api', __name__, url_prefix='/api', template_folder="")
 
+from tentacles.Services.Interfaces.web_interface.api import exchanges
 from tentacles.Services.Interfaces.web_interface.api import metadata
 from tentacles.Services.Interfaces.web_interface.api import trading
 from tentacles.Services.Interfaces.web_interface.api import user_commands
 
 
+from tentacles.Services.Interfaces.web_interface.api.exchanges import (
+    is_compatible_account,
+)
 from tentacles.Services.Interfaces.web_interface.api.metadata import (
     version,
     upgrade_version,
@@ -40,6 +44,7 @@
 
 
 __all__ = [
+    "is_compatible_account",
     "version",
     "upgrade_version",
     "user_feedback",

Services/Interfaces/web_interface/api/exchanges.py

@@ -0,0 +1,30 @@
+#  Drakkar-Software OctoBot-Interfaces
+#  Copyright (c) Drakkar-Software, All rights reserved.
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 3.0 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library.
+import flask
+
+import tentacles.Services.Interfaces.web_interface.api as api
+import tentacles.Services.Interfaces.web_interface.login as login
+import tentacles.Services.Interfaces.web_interface.models as models
+
+
+@api.api.route("/is_compatible_account", methods=['POST'])
+@login.login_required_when_activated
+def is_compatible_account():
+    request_data = flask.request.get_json()
+    return flask.jsonify(models.is_compatible_account(request_data["exchange"],
+                                                      request_data["apiKey"],
+                                                      request_data["apiSecret"],
+                                                      request_data["apiPassword"]))

Services/Interfaces/web_interface/controllers/backtesting.py

@@ -73,7 +73,9 @@ def data_collector():
             success, reply = models.get_delete_data_file(file)
         elif action_type == "start_collector":
             details = flask.request.get_json()
-            success, reply = models.collect_data_file(details["exchange"], details["symbol"])
+            success, reply = models.collect_data_file(details["exchange"], details["symbol"],
+                                                      details["startTimestamp"], details["endTimestamp"],
+                                                      bool(details["startTimestamp"]))
         elif action_type == "import_data_file":
             if flask.request.files:
                 file = flask.request.files['file']
@@ -87,7 +89,8 @@ def data_collector():
             # here return template to force page reload because of file upload via input form
             return flask.render_template('data_collector.html',
                                          data_files=models.get_data_files_with_description(),
-                                         ccxt_exchanges=sorted(models.get_full_exchange_list()),
+                                         other_ccxt_exchanges=sorted(models.get_other_history_exchange_list()),
+                                         full_candle_history_ccxt_exchanges=models.full_candle_history_ccxt_exchanges(),
                                          current_exchange=models.get_current_exchange(),
                                          full_symbol_list=sorted(models.get_symbol_list([current_exchange])),
                                          alert=alert)
@@ -112,7 +115,8 @@ def data_collector():
         current_exchange = models.get_current_exchange()
         return flask.render_template('data_collector.html',
                                      data_files=models.get_data_files_with_description(),
-                                     ccxt_exchanges=sorted(models.get_full_exchange_list()),
+                                     other_ccxt_exchanges=sorted(models.get_other_history_exchange_list()),
+                                     full_candle_history_ccxt_exchanges=models.get_full_candle_history_exchange_list(),
                                      current_exchange=models.get_current_exchange(),
                                      full_symbol_list=sorted(models.get_symbol_list([current_exchange])),
                                      origin_page=origin_page,