rl-actor-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "%pylab inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "code_folding": [
     0
    ],
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "# imports \n",
    "import gym\n",
    "import sys\n",
    "sys.path.append('/home/peter/code/projects')\n",
    "sys.path.append('C:\\\\users\\\\spook\\\\dropbox\\\\code\\\\projects')\n",
    "from gym import error, spaces, utils\n",
    "from gym.utils import seeding\n",
    "from collections import Counter\n",
    "import time\n",
    "import progressbar as pb\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "import shutil\n",
    "import tensorflow as tf\n",
    "#import aidevutil.denoise as denoise\n",
    "from empyrical import sortino_ratio, calmar_ratio, omega_ratio\n",
    "from stable_baselines.common.policies import MlpPolicy, MlpLstmPolicy, ActorCriticPolicy, FeedForwardPolicy\n",
    "from stable_baselines.common.vec_env import SubprocVecEnv, VecEnv, VecEnvWrapper\n",
    "from stable_baselines import A2C, PPO2, DQN, ACKTR, ACER\n",
    "from stable_baselines.common.vec_env import DummyVecEnv\n",
    "from trading_env import TradingEnv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "np.random.seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "dataname = 'eurusd_1m_2015'\n",
    "\n",
    "input_source = np.load(\"data/data_%s.npy\" % dataname)\n",
    "to_predict = np.load(\"data/data_%s_targets.npy\" % dataname)\n",
    "\n",
    "to_predict = to_predict[3,:].reshape(-1)\n",
    "\n",
    "input_source = input_source.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "input_source.shape, to_predict.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# lookahead bias on purpose\n",
    "#input_source[:, 22] = np.roll(input_source[:, 22], -3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest, f_regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#k = SelectKBest(score_func=f_regression, k=16)\n",
    "#k.fit(input_source, to_predict)\n",
    "#input_source = k.transform(input_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "is_orig = np.copy(input_source)\n",
    "cp = int(0.8*len(input_source))\n",
    "test_input_source = input_source[cp:, :]\n",
    "test_to_predict = to_predict[cp:]\n",
    "input_source = input_source[0:cp, :]\n",
    "to_predict = to_predict[0:cp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "input_source.shape, to_predict.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "bars_per_episode = 1000\n",
    "winlen = 1\n",
    "traded_amt = 100\n",
    "initial_balance = 1000000000000\n",
    "commission = 0.0\n",
    "slippage = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# multiprocess environment\n",
    "n_cpu = 4\n",
    "env = SubprocVecEnv([lambda: TradingEnv(input_source, to_predict,\n",
    "                 winlen=winlen, bars_per_episode=bars_per_episode, traded_amt=traded_amt, initial_balance=initial_balance,\n",
    "                 commission=commission, slippage=slippage,\n",
    "                 reward_type='cur_balance',\n",
    "                 min_ratio_trades = 20,\n",
    "                 max_position_time = 30,\n",
    "                 ) for i in range(n_cpu)])\n",
    "#env = TradingEnv(input_source, to_predict,\n",
    "#                 winlen=winlen, bars_per_episode=bars_per_episode, traded_amt=traded_amt, initial_balance=initial_balance,\n",
    "#                 commission=commission, slippage=slippage,\n",
    "#                 reward_type='sortino',\n",
    "#                 min_ratio_trades = 50,\n",
    "#                 )\n",
    "#env = DummyVecEnv([lambda: env])18\n",
    "\n",
    "t = 0\n",
    "# Custom MLP policy of two layers of size 32 each with tanh activation function\n",
    "#policy_kwargs = dict(act_fun=tf.nn.tanh, net_arch=[128, 128])\n",
    "\n",
    "#[shutil.rmtree('/home/peter/tblog/'+x) for x in os.listdir('/home/peter/tblog/') if x]\n",
    "# model = PPO2(MlpPolicy, env, n_steps=32, verbose=0, #nminibatches=1, \n",
    "#              policy_kwargs=policy_kwargs, \n",
    "#              gamma=0.99,#0.99,\n",
    "#              ent_coef=0.01,#0.01,\n",
    "#              learning_rate=0.0005,\n",
    "#              vf_coef=0.5,\n",
    "#              max_grad_norm=0.5,\n",
    "#              lam=0.95,\n",
    "#              tensorboard_log='/home/peter/tblog')\n",
    "model = ACKTR(MlpPolicy, env, verbose=0, \n",
    "#              policy_kwargs=policy_kwargs, \n",
    "              tensorboard_log='/home/peter/tblog')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    model.learn(total_timesteps=100_000_000)\n",
    "except KeyboardInterrupt:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#model.save('model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "input_source.shape, test_input_source.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "#model.load('model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "code_folding": [
     17
    ]
   },
   "outputs": [],
   "source": [
    "env = TradingEnv(test_input_source, test_to_predict,\n",
    "                 winlen=winlen, bars_per_episode=12000, traded_amt=traded_amt,\n",
    "                 commission=commission, slippage=slippage,\n",
    "                 initial_balance=initial_balance,\n",
    "                 reward_type='cur_balance',\n",
    "                 min_ratio_trades = 20,\n",
    "                 max_position_time = 30,\n",
    "                 )\n",
    "env = DummyVecEnv([lambda: env])\n",
    "\n",
    "\n",
    "nstate = model.initial_state  # get the initial state vector for the reccurent network\n",
    "#dones = np.zeros(nstate.shape[0])  # set all environment to not done\n",
    "#nstate=None\n",
    "\n",
    "observation = env.envs[0].reset()#env.reset()\n",
    "done = False\n",
    "navs = []\n",
    "acts = []\n",
    "for i in tqdm(range(env.envs[0].bars_per_episode)):\n",
    "    action, nstate = model.predict([observation], state=nstate, deterministic=1)\n",
    "    acts.append(action)\n",
    "    observation, reward, done, info = env.envs[0].step(action)#env.step(action)\n",
    "    if done:\n",
    "        break\n",
    "navs = env.envs[0].returns\n",
    "kl = []\n",
    "t = 0\n",
    "for n in navs:\n",
    "    t = t + n\n",
    "    kl.append(t)\n",
    "plot(kl);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "len(navs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false",
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# calculate the likelihood of success for any given episode\n",
    "try:\n",
    "    l = 10000\n",
    "\n",
    "    krl = []\n",
    "    p = pb.ProgressBar(max_value=l)\n",
    "    for i in range(l):\n",
    "        p.update(i)\n",
    "        observation = env.envs[0].reset()\n",
    "        nstate = model.initial_state\n",
    "        done = False\n",
    "        navs = []\n",
    "        for i in (range(env.envs[0].bars_per_episode)):\n",
    "            action, nstate = model.predict([observation], state=nstate, deterministic=1)\n",
    "            acts.append(action)\n",
    "            observation, reward, done, info = env.envs[0].step(action)#env.step(action)\n",
    "            if done:\n",
    "                break\n",
    "        krl.append(sum(env.envs[0].returns ))\n",
    "    p.finish()\n",
    "except KeyboardInterrupt:\n",
    "    pass\n",
    "\n",
    "krl = np.array(krl)\n",
    "print('Profit likelihood: %3.3f%%' % (100*(sum(krl > 0) / len(krl))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "hist(krl,50);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "sum(env.envs[0].returns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}