Merge pull request #1043 from Drakkar-Software/dev

Master merge

Services/Interfaces/web_interface/api/__init__.py

@@ -24,6 +24,11 @@
 import tentacles.Services.Interfaces.web_interface.api.bots
 import tentacles.Services.Interfaces.web_interface.api.webhook
 
+from tentacles.Services.Interfaces.web_interface.api.webhook import (
+    has_webhook,
+    register_webhook
+)
+
 
 def register():
     blueprint = flask.Blueprint('api', __name__, url_prefix='/api', template_folder="")
@@ -41,5 +46,7 @@ def register():
 
 
 __all__ = [
+    "has_webhook",
+    "register_webhook",
     "register",
 ]

Trading/Exchange/kucoin/kucoin_exchange.py

@@ -17,13 +17,12 @@
 import time
 import decimal
 import typing
+import ccxt
 
 import octobot_commons.logging as logging
 import octobot_trading.errors
 import octobot_trading.exchanges as exchanges
 import octobot_trading.exchanges.connectors.ccxt.enums as ccxt_enums
-import octobot_trading.exchanges.connectors.ccxt.constants as ccxt_constants
-import octobot_trading.exchanges.connectors.ccxt.ccxt_client_util as ccxt_client_util
 import octobot_commons.constants as commons_constants
 import octobot_trading.constants as constants
 import octobot_trading.enums as trading_enums
@@ -121,6 +120,51 @@ def get_supported_exchange_types(cls) -> list:
             trading_enums.ExchangeTypes.FUTURE,
         ]
 
+    async def get_account_id(self, **kwargs: dict) -> str:
+        # It is currently impossible to fetch subaccounts account id, use a constant value to identify it.
+        # updated: 23/08/2023
+        try:
+            account_id = None
+            subaccount_id = None
+            sub_accounts = await self.connector.client.private_get_sub_accounts()
+            for account in sub_accounts["data"]:
+                if account["subUserId"]:
+                    subaccount_id = account["subName"]
+                else:
+                    # only subaccounts have a subUserId: if this condition is True, we are on the main account
+                    account_id = account["subName"]
+            if subaccount_id:
+                # there is at least a subaccount: ensure the current account is the main account as there is no way
+                # to know the id of the current account (only a list of existing accounts)
+                subaccount_api_key_details = await self.connector.client.private_get_sub_api_key(
+                    {"subName": subaccount_id}
+                )
+                if "data" not in subaccount_api_key_details or "msg" in subaccount_api_key_details:
+                    # subaccounts can't fetch other accounts data, if this is False, we are on a subaccount
+                    self.logger.error(
+                        f"kucoin api changed: it is now possible to call private_get_sub_accounts on subaccounts. "
+                        f"kucoin get_account_id has to be updated. "
+                        f"sub_accounts={sub_accounts} subaccount_api_key_details={subaccount_api_key_details}"
+                    )
+                    return constants.DEFAULT_ACCOUNT_ID
+            if account_id is None:
+                self.logger.error(
+                    f"kucoin api changed: can't fetch master account account_id. "
+                    f"kucoin get_account_id has to be updated."
+                    f"sub_accounts={sub_accounts}"
+                )
+                account_id = constants.DEFAULT_ACCOUNT_ID
+            # we are on the master account
+            return account_id
+        except ccxt.AuthenticationError:
+            # when api key is wrong
+            raise
+        except ccxt.ExchangeError:
+            # ExchangeError('kucoin This user is not a master user')
+            # raised when calling this endpoint with a subaccount
+            return constants.DEFAULT_ACCOUNT_ID
+
+
     @_kucoin_retrier
     async def get_symbol_prices(self, symbol, time_frame, limit: int = 200, **kwargs: dict):
         if "since" in kwargs:
@@ -198,10 +242,17 @@ async def set_symbol_leverage(self, symbol: str, leverage: float, **kwargs):
 
     @_kucoin_retrier
     async def get_open_orders(self, symbol=None, since=None, limit=None, **kwargs) -> list:
+        if limit is None:
+            # default is 50, The maximum cannot exceed 1000
+            # https://www.kucoin.com/docs/rest/futures-trading/orders/get-order-list
+            limit = 200
         regular_orders = await super().get_open_orders(symbol=symbol, since=since, limit=limit, **kwargs)
-        # add untriggered stop orders (different api endpoint)
-        kwargs["stop"] = True
-        stop_orders = await super().get_open_orders(symbol=symbol, since=since, limit=limit, **kwargs)
+        stop_orders = []
+        if self.exchange_manager.is_future:
+            # stop ordes are futures only for now
+            # add untriggered stop orders (different api endpoint)
+            kwargs["stop"] = True
+            stop_orders = await super().get_open_orders(symbol=symbol, since=since, limit=limit, **kwargs)
         return regular_orders + stop_orders
 
     @_kucoin_retrier

Trading/Exchange/okx/okx_exchange.py

@@ -167,6 +167,14 @@ def get_supported_exchange_types(cls) -> list:
     def _fix_limit(self, limit: int) -> int:
         return min(self.MAX_PAGINATION_LIMIT, limit) if limit else limit
 
+    async def get_account_id(self, **kwargs: dict) -> str:
+        accounts = await self.connector.client.fetch_accounts()
+        try:
+            return accounts[0]["id"]
+        except IndexError:
+            # should never happen as at least one account should be available
+            return None
+
     async def get_sub_account_list(self):
         sub_account_list = (await self.connector.client.privateGetUsersSubaccountList()).get("data", [])
         if not sub_account_list:
@@ -232,9 +240,11 @@ async def _get_all_typed_orders(self, method, symbol=None, since=None, limit=Non
             return regular_orders
         # add order types of order (different param in api endpoint)
         other_orders = []
-        for order_type in self._get_used_order_types():
-            kwargs["ordType"] = order_type
-            other_orders += await method(symbol=symbol, since=since, limit=limit, **kwargs)
+        if self.exchange_manager.is_future:
+            # stop orders are futures only for now
+            for order_type in self._get_used_order_types():
+                kwargs["ordType"] = order_type
+                other_orders += await method(symbol=symbol, since=since, limit=limit, **kwargs)
         return regular_orders + other_orders
 
     async def get_open_orders(self, symbol=None, since=None, limit=None, **kwargs) -> list:
@@ -290,7 +300,7 @@ async def _verify_order(self, created_order, order_type, symbol, price, side, ge
 
     def _is_oco_order(self, params):
         return all(
-            oco_order_param in params
+            oco_order_param in (params or {})
             for oco_order_param in (
                 self.connector.adapter.OKX_STOP_LOSS_PRICE,
                 self.connector.adapter.OKX_TAKE_PROFIT_PRICE

Trading/Mode/grid_trading_mode/grid_trading.py

@@ -291,6 +291,9 @@ def read_config(self):
         if self.allow_order_funds_redispatch:
             # check every day that funds should not be redispatched and of orders are missing
             self.health_check_interval_secs = commons_constants.DAYS_TO_SECONDS
+        self.compensate_for_missed_mirror_order = self.symbol_trading_config.get(
+            self.trading_mode.COMPENSATE_FOR_MISSED_MIRROR_ORDER, self.compensate_for_missed_mirror_order
+        )
 
     async def _handle_staggered_orders(self, current_price, ignore_mirror_orders_only, ignore_available_funds):
         self._init_allowed_price_ranges(current_price)
@@ -346,11 +349,24 @@ async def _generate_staggered_orders(self, current_price, ignore_available_funds
         existing_orders = order_manager.get_open_orders(self.symbol)
 
         sorted_orders = self._get_grid_trades_or_orders(existing_orders)
-        recent_trades_time = trading_api.get_exchange_current_time(
-            self.exchange_manager) - self.RECENT_TRADES_ALLOWED_TIME
-        recently_closed_trades = trading_api.get_trade_history(self.exchange_manager, symbol=self.symbol,
-                                                               since=recent_trades_time)
-        recently_closed_trades = self._get_grid_trades_or_orders(recently_closed_trades)
+        oldest_existing_order_creation_time = min(
+            order.creation_time for order in sorted_orders
+        ) if sorted_orders else 0
+        recent_trades_time = max(
+            trading_api.get_exchange_current_time(
+                self.exchange_manager
+            ) - self.RECENT_TRADES_ALLOWED_TIME,
+            oldest_existing_order_creation_time
+        )
+        # list of trades orders from the most recent one to the oldest one
+        recently_closed_trades = sorted([
+            trade
+            for trade in trading_api.get_trade_history(
+                self.exchange_manager, symbol=self.symbol, since=recent_trades_time
+            )
+            # non limit orders are not to be taken into account
+            if trade.trade_type in (trading_enums.TraderOrderType.BUY_LIMIT, trading_enums.TraderOrderType.SELL_LIMIT)
+        ], key=lambda t: -t.executed_time)
 
         lowest_buy = max(trading_constants.ZERO, self.buy_price_range.lower_bound)
         highest_buy = self.buy_price_range.higher_bound
@@ -397,6 +413,7 @@ async def _generate_staggered_orders(self, current_price, ignore_available_funds
         missing_orders, state, _ = self._analyse_current_orders_situation(
             sorted_orders, recently_closed_trades, lowest_buy, highest_sell, current_price
         )
+        await self._handle_missed_mirror_orders_fills(recently_closed_trades, missing_orders, current_price)
         try:
             buy_orders = self._create_orders(lowest_buy, highest_buy,
                                              trading_enums.TradeOrderSide.BUY, sorted_orders,
@@ -406,6 +423,7 @@ async def _generate_staggered_orders(self, current_price, ignore_available_funds
                                               trading_enums.TradeOrderSide.SELL, sorted_orders,
                                               current_price, missing_orders, state, self.sell_funds, ignore_available_funds,
                                               recently_closed_trades)
+
             if state is self.FILL:
                 self._ensure_used_funds(buy_orders, sell_orders, sorted_orders, recently_closed_trades)
         except staggered_orders_trading.ForceResetOrdersException:

Trading/Mode/grid_trading_mode/tests/test_grid_trading_mode.py

@@ -14,7 +14,6 @@
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
 import contextlib
-
 import numpy
 import pytest
 import os.path
@@ -105,6 +104,7 @@ async def _get_tools(symbol, btc_holdings=None, additional_portfolio={}, fees=No
         producer.flat_increment = decimal.Decimal(5)
         producer.buy_orders_count = 25
         producer.sell_orders_count = 25
+        producer.compensate_for_missed_mirror_order = True
         test_trading_modes.set_ready_to_start(producer)
 
         yield producer, mode.get_trading_mode_consumers()[0], exchange_manager
@@ -769,6 +769,110 @@ async def test_start_after_offline_buy_side_10_filled():
         _check_created_orders(producer, trading_api.get_open_orders(exchange_manager), 100)
 
 
+async def test_start_after_offline_x_filled_and_price_back_should_sell_to_recreate_buy():
+    symbol = "BTC/USDT"
+    async with _get_tools(symbol) as (producer, _, exchange_manager):
+        orders_count = 25 + 25
+
+        price = decimal.Decimal(200)
+        trading_api.force_set_mark_price(exchange_manager, producer.symbol, price)
+        await producer._ensure_staggered_orders()
+        await asyncio.create_task(_check_open_orders_count(exchange_manager, orders_count))
+        original_orders = copy.copy(trading_api.get_open_orders(exchange_manager))
+        assert len(original_orders) == orders_count
+        pre_btc_portfolio = trading_api.get_portfolio_currency(exchange_manager, "BTC").available
+        pre_usdt_portfolio = trading_api.get_portfolio_currency(exchange_manager, "USDT").available
+
+        # offline simulation: orders get filled but not replaced => price moved to 150
+        open_orders = trading_api.get_open_orders(exchange_manager)
+        offline_filled = [
+            o
+            for o in open_orders
+            if o.side == trading_enums.TradeOrderSide.BUY and o.origin_price >= decimal.Decimal("150")
+        ]
+        # this is 10 orders
+        assert len(offline_filled) == 10
+        max_filled_order_price = max(o.origin_price for o in offline_filled)
+        assert max_filled_order_price == decimal.Decimal(195)
+        for order in offline_filled:
+            await _fill_order(order, exchange_manager, trigger_update_callback=False, producer=producer)
+        post_btc_portfolio = trading_api.get_portfolio_currency(exchange_manager, "BTC").available
+        post_usdt_portfolio = trading_api.get_portfolio_currency(exchange_manager, "USDT").available
+        # buy orders filled: BTC increased
+        assert pre_btc_portfolio < post_btc_portfolio
+        # no sell order filled, available USDT is constant
+        assert pre_usdt_portfolio == post_usdt_portfolio
+        assert len(trading_api.get_open_orders(exchange_manager)) == orders_count - len(offline_filled)
+
+        # back online: restore orders according to current price
+        # simulate current price as back to 180: should quickly sell BTC bought between 150 and 180 to be able to
+        # create buy orders between 150 and 180
+        price = decimal.Decimal(180)
+        trading_api.force_set_mark_price(exchange_manager, producer.symbol, price)
+        await producer._ensure_staggered_orders()
+        # restored orders
+        await asyncio.create_task(_check_open_orders_count(exchange_manager, orders_count))
+        assert 0 <= trading_api.get_portfolio_currency(exchange_manager, "BTC").available <= post_btc_portfolio
+        open_orders = trading_api.get_open_orders(exchange_manager)
+        # created 4 additional sell orders
+        assert len([order for order in open_orders if order.side is trading_enums.TradeOrderSide.SELL]) == 25 + 4
+        # restored 6 out of 10 filled buy orders
+        assert len([order for order in open_orders if order.side is trading_enums.TradeOrderSide.BUY]) == 25 - 10 + 6
+        _check_created_orders(producer, trading_api.get_open_orders(exchange_manager), 200)
+
+
+async def test_start_after_offline_x_filled_and_price_back_should_buy_to_recreate_sell():
+    symbol = "BTC/USDT"
+    async with _get_tools(symbol) as (producer, _, exchange_manager):
+        orders_count = 25 + 25
+
+        price = decimal.Decimal(200)
+        trading_api.force_set_mark_price(exchange_manager, producer.symbol, price)
+        await producer._ensure_staggered_orders()
+        await asyncio.create_task(_check_open_orders_count(exchange_manager, orders_count))
+        original_orders = copy.copy(trading_api.get_open_orders(exchange_manager))
+        assert len(original_orders) == orders_count
+        pre_btc_portfolio = trading_api.get_portfolio_currency(exchange_manager, "BTC").available
+        pre_usdt_portfolio = trading_api.get_portfolio_currency(exchange_manager, "USDT").available
+
+        # offline simulation: orders get filled but not replaced => price moved to 150
+        open_orders = trading_api.get_open_orders(exchange_manager)
+        offline_filled = [
+            o
+            for o in open_orders
+            if o.side == trading_enums.TradeOrderSide.SELL and o.origin_price <= decimal.Decimal("250")
+        ]
+        # this is 10 orders
+        assert len(offline_filled) == 10
+        max_filled_order_price = max(o.origin_price for o in offline_filled)
+        assert max_filled_order_price == decimal.Decimal(250)
+        for order in offline_filled:
+            await _fill_order(order, exchange_manager, trigger_update_callback=False, producer=producer)
+        post_btc_portfolio = trading_api.get_portfolio_currency(exchange_manager, "BTC").available
+        post_usdt_portfolio = trading_api.get_portfolio_currency(exchange_manager, "USDT").available
+        # buy orders filled: available BTC is constant
+        assert pre_btc_portfolio == post_btc_portfolio
+        # no sell order filled, available USDT increased
+        assert pre_usdt_portfolio <= post_usdt_portfolio
+        assert len(trading_api.get_open_orders(exchange_manager)) == orders_count - len(offline_filled)
+
+        # back online: restore orders according to current price
+        # simulate current price as back to 220: should quickly buy BTC sold between 250 and 220 to be able to
+        # create sell orders between 220 and 250
+        price = decimal.Decimal(220)
+        trading_api.force_set_mark_price(exchange_manager, producer.symbol, price)
+        await producer._ensure_staggered_orders()
+        # restored orders
+        await asyncio.create_task(_check_open_orders_count(exchange_manager, orders_count))
+        assert 0 <= trading_api.get_portfolio_currency(exchange_manager, "USDT").available <= post_usdt_portfolio
+        open_orders = trading_api.get_open_orders(exchange_manager)
+        # restored 6 out of 10 sell orders
+        assert len([order for order in open_orders if order.side is trading_enums.TradeOrderSide.SELL]) == 25 - 10 + 6
+        # created 4 additional buy orders
+        assert len([order for order in open_orders if order.side is trading_enums.TradeOrderSide.BUY]) == 25 + 4
+        _check_created_orders(producer, trading_api.get_open_orders(exchange_manager), 200)
+
+
 async def test_start_after_offline_with_added_funds_increasing_orders_count():
     symbol = "BTC/USDT"
     async with _get_tools(symbol) as (producer, _, exchange_manager):