clean unused import and some refactor (safe code)

.gitignore

@@ -347,9 +347,7 @@ backtest_results/
 historical_data/
 
 # Config files
-!api_key_secrets/binance/example_user.json
-!api_key_secrets/bybit/example_user.json
-api_key_secrets/*
+api-keys.json
 !live_settings/binance/default.json
 !live_settings/bybit/default.json
 live_settings/*

backtest.py

@@ -5,11 +5,11 @@
 
 import matplotlib.pyplot as plt
 import nevergrad as ng
+import pandas as pd
 import ray
 from ray import tune
 from ray.tune.schedulers import AsyncHyperBandScheduler
 from ray.tune.suggest import ConcurrencyLimiter
-# from ray.tune.suggest.hyperopt import HyperOptSearch
 from ray.tune.suggest.nevergrad import NevergradSearch
 
 from downloader import Downloader, prep_backtest_config
@@ -152,7 +152,7 @@ def backtest(config: dict, ticks: np.ndarray, return_fills=False, do_print=False
         long_pnl_f = calc_long_pnl_inverse
         shrt_pnl_f = calc_shrt_pnl_inverse
         cost_f = calc_cost_inverse
-        iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice, \
+        iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
                               highest_bid, lowest_ask, last_price, do_long, do_shrt: \
             iter_entries_inverse(config['price_step'], config['qty_step'], config['min_qty'],
                                  config['min_cost'], config['ddown_factor'], config['qty_pct'],
@@ -185,7 +185,7 @@ def backtest(config: dict, ticks: np.ndarray, return_fills=False, do_print=False
         long_pnl_f = calc_long_pnl_linear
         shrt_pnl_f = calc_shrt_pnl_linear
         cost_f = calc_cost_linear
-        iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice, \
+        iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
                               highest_bid, lowest_ask, last_price, do_long, do_shrt: \
             iter_entries_linear(config['price_step'], config['qty_step'], config['min_qty'],
                                 config['min_cost'], config['ddown_factor'], config['qty_pct'],

binance.py

@@ -1,27 +1,22 @@
 import asyncio
+import hashlib
+import hmac
 import json
-import websockets
-import os
 import sys
-import numpy as np
-import pandas as pd
-import pprint
-import datetime
-import aiohttp
-import hmac
-import hashlib
+from time import time
 from urllib.parse import urlencode
-from math import ceil
-from math import floor
-from time import time, sleep
-from typing import Callable, Iterator
-from passivbot import load_key_secret, load_live_settings, make_get_filepath, print_, \
-    ts_to_date, flatten, filter_orders, Bot, start_bot, round_up, round_dn, \
-    calc_min_order_qty, sort_dict_keys, \
-    iter_long_closes_linear, iter_shrt_closes_linear, calc_ema, iter_entries_linear, \
+
+import aiohttp
+import numpy as np
+import websockets
+
+from passivbot import load_key_secret, load_live_settings, print_, \
+    ts_to_date, flatten, Bot, start_bot, round_up, calc_min_order_qty, sort_dict_keys, \
+    iter_long_closes_linear, iter_shrt_closes_linear, iter_entries_linear, \
     iter_long_closes_inverse, iter_shrt_closes_inverse, calc_ema, iter_entries_inverse, \
     calc_cost_linear, calc_cost_inverse
 
+
 async def create_bot(user: str, settings: str):
     bot = BinanceBot(user, settings)
     await bot._init()
@@ -38,7 +33,6 @@ def __init__(self, user: str, settings: dict):
         self.base_endpoint = ''
         self.key, self.secret = load_key_secret('binance', user)
 
-
     async def public_get(self, url: str, params: dict = {}) -> dict:
         async with self.session.get(self.base_endpoint + url, params=params) as response:
             result = await response.text()
@@ -54,8 +48,8 @@ async def private_(self, type_: str, url: str, params: dict = {}) -> dict:
                 params[k] = str(params[k])
         params = sort_dict_keys(params)
         params['signature'] = hmac.new(self.secret.encode('utf-8'),
-                                  urlencode(params).encode('utf-8'),
-                                  hashlib.sha256).hexdigest()
+                                       urlencode(params).encode('utf-8'),
+                                       hashlib.sha256).hexdigest()
         headers = {'X-MBX-APIKEY': self.key}
         async with getattr(self.session, type_)(self.base_endpoint + url, params=params,
                                                 headers=headers) as response:
@@ -102,17 +96,15 @@ def init_market_type(self):
                                         self.markup_range, self.n_close_orders, balance, pos_size,
                                         pos_price, highest_bid)
 
-
-            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice, \
-                liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
+            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
+                                       liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
                 iter_entries_linear(self.price_step, self.qty_step, self.min_qty, self.min_cost,
                                     self.ddown_factor, self.qty_pct, self.leverage,
                                     self.grid_spacing, self.grid_coefficient, self.ema_spread,
                                     self.stop_loss_liq_diff, self.stop_loss_pos_pct, balance,
                                     long_psize, long_pprice, shrt_psize, shrt_pprice, liq_price,
                                     highest_bid, lowest_ask, ema, last_price, do_long, do_shrt)
 
-
             self.cost_f = calc_cost_linear
         else:
             print('inverse perpetual')
@@ -144,8 +136,8 @@ def init_market_type(self):
                                          self.markup_range, self.n_close_orders, balance, pos_size,
                                          pos_price, highest_bid)
 
-            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice, \
-                liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
+            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
+                                       liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
                 iter_entries_inverse(self.price_step, self.qty_step, self.min_qty, self.min_cost,
                                      self.ddown_factor, self.qty_pct, self.leverage,
                                      self.grid_spacing, self.grid_coefficient, self.ema_spread,
@@ -195,7 +187,7 @@ async def _init(self):
         max_lev = 0
         for e in leverage_bracket:
             if ('pair' in e and e['pair'] == self.pair) or \
-                    ('symbol' in e and e['symbol'] ==self.symbol):
+                    ('symbol' in e and e['symbol'] == self.symbol):
                 for br in e['brackets']:
                     max_lev = max(max_lev, int(br['initialLeverage']))
                 break
@@ -213,7 +205,7 @@ async def init_ema(self):
         ticks = sorted(ticks + additional_ticks, key=lambda x: x['trade_id'])
         ema = ticks[0]['price']
         for i in range(1, len(ticks)):
-            if ticks[i]['price'] != ticks[i-1]['price']:
+            if ticks[i]['price'] != ticks[i - 1]['price']:
                 ema = ema * self.ema_alpha_ + ticks[i]['price'] * self.ema_alpha
         self.ema = ema
 
@@ -233,11 +225,11 @@ async def check_if_other_positions(self, abort=True):
                     do_abort = True
         for e in open_orders:
             if e['symbol'] != self.symbol:
-                    print('\n\nWARNING\n\n')
-                    print('account has open orders in other symbol:', e)
-                    print('\naborting')
-                    print('\n\n')
-                    do_abort = True
+                print('\n\nWARNING\n\n')
+                print('account has open orders in other symbol:', e)
+                print('\naborting')
+                print('\n\n')
+                do_abort = True
         if do_abort:
             if abort:
                 raise Exception('please close other positions and cancel other open orders')
@@ -321,7 +313,7 @@ async def fetch_position(self) -> dict:
             if e['asset'] == (self.quot if self.market_type == 'linear_perpetual' else self.coin):
                 position['wallet_balance'] = float(e['balance']) / self.contract_size
                 position['equity'] = position['wallet_balance'] + float(e['crossUnPnl']) / self.contract_size
-                #position['available_margin'] = float(e['availableBalance']) / self.contract_size
+                # position['available_margin'] = float(e['availableBalance']) / self.contract_size
                 # position['available_balance'] = float(e['availableBalance'])
                 break
         return position
@@ -434,4 +426,3 @@ async def main() -> None:
 
 if __name__ == '__main__':
     asyncio.run(main())
-

bybit.py

@@ -1,27 +1,22 @@
 import asyncio
+import hashlib
+import hmac
 import json
-import websockets
-import os
 import sys
+from time import time
+from urllib.parse import urlencode
+
+import aiohttp
 import numpy as np
-import pandas as pd
-import pprint
-import hashlib
-import hmac
+import websockets
 from dateutil import parser
-from math import ceil
-from math import floor
-from time import time, sleep
-from typing import Callable, Iterator
-from passivbot import load_key_secret, load_live_settings, make_get_filepath, print_, \
-    ts_to_date, flatten, Bot, start_bot, round_up, round_dn, \
-    calc_min_order_qty_inverse, sort_dict_keys, calc_ema, calc_diff, \
-    iter_long_closes_inverse, iter_shrt_closes_inverse, iter_entries_inverse, \
+
+from passivbot import load_key_secret, load_live_settings, print_, \
+    ts_to_date, flatten, Bot, start_bot, calc_min_order_qty_inverse, sort_dict_keys, calc_ema, iter_long_closes_inverse, \
+    iter_shrt_closes_inverse, iter_entries_inverse, \
     iter_long_closes_linear, iter_shrt_closes_linear, iter_entries_linear, calc_long_pnl_linear, \
     calc_long_pnl_inverse, calc_shrt_pnl_linear, \
     calc_shrt_pnl_inverse, calc_cost_linear, calc_cost_inverse
-import aiohttp
-from urllib.parse import urlencode
 
 
 def first_capitalized(s: str):
@@ -53,7 +48,6 @@ def format_tick(tick: dict) -> dict:
 
 
 async def fetch_ticks(cc, symbol: str, from_id: int = None, do_print=True) -> [dict]:
-
     params = {'symbol': symbol, 'limit': 1000}
     if from_id:
         params['from'] = max(0, from_id)
@@ -68,9 +62,11 @@ async def fetch_ticks(cc, symbol: str, from_id: int = None, do_print=True) -> [d
                 ts_to_date(trades[0]['timestamp'] / 1000)])
     return trades
 
+
 def date_to_ts(date: str):
     return parser.parse(date).timestamp() * 1000
 
+
 async def create_bot(user: str, settings: str):
     bot = Bybit(user, settings)
     await bot._init()
@@ -112,8 +108,8 @@ def init_market_type(self):
                                         self.markup_range, self.n_close_orders, balance, pos_size,
                                         pos_price, highest_bid)
 
-            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice, \
-                liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
+            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
+                                       liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
                 iter_entries_linear(self.price_step, self.qty_step, self.min_qty, self.min_cost,
                                     self.ddown_factor, self.qty_pct, self.leverage,
                                     self.grid_spacing, self.grid_coefficient, self.ema_spread,
@@ -164,28 +160,33 @@ def init_market_type(self):
                                          self.markup_range, self.n_close_orders, balance, pos_size,
                                          pos_price, highest_bid)
 
-            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice, \
-                liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
+            self.iter_entries = lambda balance, long_psize, long_pprice, shrt_psize, shrt_pprice,
+                                       liq_price, highest_bid, lowest_ask, ema, last_price, do_long, do_shrt: \
                 iter_entries_inverse(self.price_step, self.qty_step, self.min_qty, self.min_cost,
                                      self.ddown_factor, self.qty_pct, self.leverage,
                                      self.grid_spacing, self.grid_coefficient, self.ema_spread,
                                      self.stop_loss_liq_diff, self.stop_loss_pos_pct, balance,
                                      long_psize, long_pprice, shrt_psize, shrt_pprice, liq_price,
                                      highest_bid, lowest_ask, ema, last_price, do_long, do_shrt)
 
-
         self.endpoints['balance'] = '/v2/private/wallet/balance'
 
     def determine_pos_side(self, o: dict) -> str:
         side = o['side'].lower()
         if side == 'buy':
-            if 'entry' in o['order_link_id']:   position_side = 'long'
-            elif 'close' in o['order_link_id']: position_side = 'shrt'
-            else:                               position_side = 'unknown'
+            if 'entry' in o['order_link_id']:
+                position_side = 'long'
+            elif 'close' in o['order_link_id']:
+                position_side = 'shrt'
+            else:
+                position_side = 'unknown'
         else:
-            if 'entry' in o['order_link_id']:   position_side = 'shrt'
-            elif 'close' in o['order_link_id']: position_side = 'long'
-            else:                               position_side = 'both'
+            if 'entry' in o['order_link_id']:
+                position_side = 'shrt'
+            elif 'close' in o['order_link_id']:
+                position_side = 'long'
+            else:
+                position_side = 'both'
         return position_side
 
     async def _init(self):
@@ -220,7 +221,7 @@ async def init_ema(self):
         ticks = sorted(ticks + additional_ticks, key=lambda x: x['trade_id'])
         ema = ticks[0]['price']
         for i in range(1, len(ticks)):
-            if ticks[i]['price'] != ticks[i-1]['price']:
+            if ticks[i]['price'] != ticks[i - 1]['price']:
                 ema = ema * self.ema_alpha_ + ticks[i]['price'] * self.ema_alpha
         self.ema = ema
 
@@ -229,7 +230,6 @@ async def init_order_book(self):
         self.ob = [float(ticker['result'][0]['bid_price']), float(ticker['result'][0]['ask_price'])]
         self.price = float(ticker['result'][0]['last_price'])
 
-
     async def fetch_open_orders(self) -> [dict]:
         fetched = await self.private_get(self.endpoints['open_orders'], {'symbol': self.symbol})
 
@@ -298,7 +298,6 @@ async def fetch_position(self) -> dict:
                 long_pos = [e['data'] for e in fetched['result'] if e['data']['position_idx'] == 1][0]
                 shrt_pos = [e['data'] for e in fetched['result'] if e['data']['position_idx'] == 2][0]
 
-
         position['long'] = {'size': float(long_pos['size']),
                             'price': float(long_pos['entry_price']),
                             'leverage': float(long_pos['leverage']),
@@ -319,7 +318,7 @@ async def fetch_position(self) -> dict:
 
     async def execute_order(self, order: dict) -> dict:
         params = {'symbol': self.symbol,
-                  'side':  first_capitalized(order['side']),
+                  'side': first_capitalized(order['side']),
                   'order_type': first_capitalized(order['type']),
                   'qty': float(order['qty']) if self.market_type == 'linear_perpetual' else int(order['qty']),
                   'close_on_trigger': False}
@@ -405,7 +404,7 @@ async def init_exchange_settings(self):
             elif self.market_type == 'inverse_perpetual':
                 res = await self.private_post('/v2/private/position/leverage/save',
                                               {'symbol': self.symbol, 'leverage': 0})
-                
+
             print(res)
         except Exception as e:
             print(e)
@@ -455,5 +454,3 @@ async def main() -> None:
 
 if __name__ == '__main__':
     asyncio.run(main())
-
-

changelog.md

@@ -160,39 +160,4 @@ All notable changes to this project will be documented in this file.
 - renamed settings["margin_limit"] to settings["balance"]
 - bug fixes and changes in trade data downloading
 - if there already is historical trade data downloaded, run the script `rename_trade_data_csvs.py` to rename all files
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+- 

downloader.py

@@ -1,6 +1,7 @@
 import gc
 
 import hjson
+import pandas as pd
 from dateutil import parser, tz
 
 from binance import create_bot as create_bot_binance