PickStocks_EvolutionaryAlgorithms.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LTSDSXZTURm5",
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:44.487080Z",
     "end_time": "2023-04-01T16:31:44.881994Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "if 'COLAB_GPU' in os.environ:\n",
    "    !git clone https://github.com/impulsecorp/PickStocks.git\n",
    "    !mv PickStocks/*.py .\n",
    "    !mv PickStocks/data .\n",
    "    !pip install -U -qq -r PickStocks/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rqKDcXZB4tNG",
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:44.553871Z",
     "end_time": "2023-04-01T16:31:46.832279Z"
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import system\n",
    "from system import *\n",
    "# small hack to prevent Colab error\n",
    "try:\n",
    "    from datablock import *\n",
    "except:\n",
    "    from datablock import *\n",
    "from deap import base, creator, tools, algorithms\n",
    "seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:46.833260Z",
     "end_time": "2023-04-01T16:31:46.878206Z"
    }
   },
   "outputs": [],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:46.849256Z",
     "end_time": "2023-04-01T16:31:47.365879Z"
    }
   },
   "outputs": [],
   "source": [
    "data_timeperiod = 'D'\n",
    "data = get_data('SPY', period=data_timeperiod, nrows=None)\n",
    "data = procdata_lite(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.366873Z",
     "end_time": "2023-04-01T16:31:47.428867Z"
    }
   },
   "outputs": [],
   "source": [
    "# for inspectiion\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#### This version of the notebook evolves more complex genomes that use thresholds instead of bins"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.429893Z",
     "end_time": "2023-04-01T16:31:47.473994Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Parameters"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.444713Z",
     "end_time": "2023-04-01T16:31:47.504810Z"
    }
   },
   "outputs": [],
   "source": [
    "system.train_set_end = 0.5 # percentage point specifying the training set end point (1.0 means all data is training set)\n",
    "system.val_set_end = 0.75    # percentage point specifying the validation set end point (1.0 means no test set)\n",
    "system.balance_data = 1\n",
    "system.scale_data = 1\n",
    "\n",
    "system.multiclass = 0\n",
    "system.multiclass_move_threshold = 1.0\n",
    "\n",
    "combine_method = 'and' # whether to combine the strategies with 'and' or 'or'\n",
    "\n",
    "individual_size = 5 # number of strategies to combine for each genome\n",
    "eval_min_trades = 100 # minumum number of trades to be eligible for reproduction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# EA Parameters\n",
    "pop_size = 128\n",
    "num_generations = 50\n",
    "survival_rate = 0.2\n",
    "crossover_prob = 0.5\n",
    "mutation_prob = 0.5\n",
    "mutation_prob_gene = 0.1\n",
    "mutation_prob_gene_hard = 0.1\n",
    "mutation_prob_gene_c = 0.5\n",
    "mutation_prob_gene_d = 0.25\n",
    "mutation_prob_gene_a = 0.0\n",
    "mutation_power = 0.2"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.460876Z",
     "end_time": "2023-04-01T16:31:47.518887Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def objective_1(a): # mean profit\n",
    "    return float(np.mean(a['profit'].values))\n",
    "def objective_2(a): # winner %\n",
    "    return float(get_winner_pct(a))\n",
    "def objective_3(a): # PF\n",
    "    return float(get_profit_factor(a))\n",
    "def objective_4(a): # num trades\n",
    "    return float(len(a))\n",
    "\n",
    "from empyrical import sortino_ratio, omega_ratio, sharpe_ratio, calmar_ratio, stability_of_timeseries\n",
    "def obj_sortino(a):\n",
    "    returns = a['profit'].values\n",
    "    return float(sortino_ratio(returns))\n",
    "\n",
    "# Combine one or more objectives here for the fitness function and set their relative weights\n",
    "objectives = [\n",
    "\n",
    "      (obj_sortino, 1.0),\n",
    "\n",
    "    # (objective_1, 0.2),\n",
    "    # (objective_2, 0.5),\n",
    "    # (objective_3, 1.0),\n",
    "\n",
    "              ]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.476988Z",
     "end_time": "2023-04-01T16:31:47.908872Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.505813Z",
     "end_time": "2023-04-01T16:31:47.908872Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.521882Z",
     "end_time": "2023-04-01T16:31:47.908872Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Base classifier"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Train LogisticRegression classifier on train data\n",
    "clf, scaler = train_classifier(LogisticRegression, data)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.537839Z",
     "end_time": "2023-04-01T16:31:47.908872Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Test on val data\n",
    "equity, pf, base_trades = qbacktest(clf, scaler, data)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:47.741294Z",
     "end_time": "2023-04-01T16:31:48.385572Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "base_trades.head()"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.387567Z",
     "end_time": "2023-04-01T16:31:48.431449Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Obtain all individual startegies (genes) to combine and their stats"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "feature_names, ranges = compute_ranges(data)\n",
    "list(zip(feature_names, ranges))"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.416489Z",
     "end_time": "2023-04-01T16:31:48.431449Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Evolutionary algorithm setup"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "worst_possible_fitness = -999999.0"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.418484Z",
     "end_time": "2023-04-01T16:31:48.461402Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def evaluate(genome):\n",
    "    return fitness_function(get_genome_alltrades_nonbinned(data, genome, base_trades, feature_names, combine_method=combine_method),\n",
    "                            objectives=objectives,\n",
    "                            eval_min_trades=eval_min_trades, worst_possible_fitness=worst_possible_fitness)\n",
    "\n",
    "def eval_wrapper(ind_list):\n",
    "    return evaluate(ind_list)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.433444Z",
     "end_time": "2023-04-01T16:31:48.507246Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def rand_gene():\n",
    "    # make new gene\n",
    "    r = rnd.randint(0, len(feature_names)-1)\n",
    "    c = rnd.uniform(ranges[r][0], ranges[r][1])\n",
    "    d = rnd.choice(['above','below','exact'])\n",
    "    a = False #rnd.choice([True, False])\n",
    "\n",
    "    return r,c,d,a\n",
    "\n",
    "def create_individual():\n",
    "    ind = []\n",
    "    for i in range(individual_size):\n",
    "        ind.append(rand_gene())\n",
    "    return ind\n",
    "\n",
    "def create_population(n):\n",
    "    return [creator.Individual(create_individual()) for _ in range(n)]\n",
    "\n",
    "def custom_mutation(individual):\n",
    "    for i in range(len(individual)):\n",
    "        if rnd.random() < mutation_prob_gene:\n",
    "            if rnd.random() < mutation_prob_gene_hard:\n",
    "                individual[i] = rand_gene()\n",
    "            else:\n",
    "                gene_r, gene_c, gene_d, gene_a = individual[i]\n",
    "\n",
    "                mag = np.abs(ranges[gene_r][1]-ranges[gene_r][0])*mutation_power\n",
    "                if rnd.random() < mutation_prob_gene_c:\n",
    "                    gene_c += rnd.uniform(-mag, mag)\n",
    "                if rnd.random() < mutation_prob_gene_d:\n",
    "                    ch = rnd.choice(['above','below','exact'])\n",
    "                    while ch == gene_d:\n",
    "                        ch = rnd.choice(['above','below','exact'])\n",
    "                    gene_d = ch\n",
    "                if rnd.random() < mutation_prob_gene_a:\n",
    "                    gene_a = not gene_a\n",
    "                individual[i] = (gene_r, gene_c, gene_d, gene_a)\n",
    "    return individual,"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.451396Z",
     "end_time": "2023-04-01T16:31:48.560105Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "creator.create(\"FitnessMax\", base.Fitness, weights=list([x[1] for x in objectives]))\n",
    "creator.create(\"Individual\", list, fitness=creator.FitnessMax)\n",
    "toolbox = base.Toolbox()\n",
    "# Register individual and population creation functions\n",
    "toolbox.register(\"individual\", create_individual)\n",
    "toolbox.register(\"population\", create_population)\n",
    "# Register genetic operators\n",
    "toolbox.register(\"mate\", tools.cxTwoPoint)\n",
    "toolbox.register(\"mutate\", custom_mutation)\n",
    "toolbox.register(\"select\", tools.selSPEA2)\n",
    "# Register the fitness function\n",
    "toolbox.register(\"evaluate\", eval_wrapper)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.463385Z",
     "end_time": "2023-04-01T16:31:48.561102Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Evolutionary algorithm"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "best_genome, best_score = run_evolution(pop_size, toolbox, num_generations, survival_rate, crossover_prob, mutation_prob, objectives, worst_possible_fitness)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:31:48.479321Z",
     "end_time": "2023-04-01T16:33:14.187361Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Description of the best strategy (feature name : [ condition to trade ])\n",
    "for i in range(len(best_genome)):\n",
    "    r,c,d,a = best_genome[i]\n",
    "    print(feature_names[r], ':', d, f'{c:.5f} {\"abs\" if a else \"\"}')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:14.189355Z",
     "end_time": "2023-04-01T16:33:14.232377Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Test the best individual on val data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "alltrades = get_genome_alltrades_nonbinned(data, best_genome, base_trades, feature_names, combine_method=combine_method)\n",
    "plt.plot(alltrades['profit'].cumsum())\n",
    "print(f'Profit factor: {get_profit_factor(alltrades):.5f}, Winners: {get_winner_pct(alltrades):.2f}%, Trades: {len(alltrades)}')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:14.204426Z",
     "end_time": "2023-04-01T16:33:14.345123Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Test the best individual on unseen data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Base test without the filter\n",
    "*_, test_trades = qbacktest(clf, scaler, data, skip_val=1, skip_test=0, quiet=1)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:14.346147Z",
     "end_time": "2023-04-01T16:33:14.854866Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Test with the filter\n",
    "alltrades = get_genome_alltrades_nonbinned(data, best_genome, test_trades, feature_names, combine_method=combine_method)\n",
    "plt.plot(alltrades['profit'].cumsum())\n",
    "print(f'Profit factor: {get_profit_factor(alltrades):.5f}, Winners: {get_winner_pct(alltrades):.2f}%, Trades: {len(alltrades)}')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:14.855864Z",
     "end_time": "2023-04-01T16:33:15.001574Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "alltrades[0:20]"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:14.997586Z",
     "end_time": "2023-04-01T16:33:15.042062Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### ------------------------------------------------------------------------------------------------------------"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:15.012593Z",
     "end_time": "2023-04-01T16:33:15.049082Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:15.028679Z",
     "end_time": "2023-04-01T16:33:15.072857Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:15.043062Z",
     "end_time": "2023-04-01T16:33:15.096853Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:15.058357Z",
     "end_time": "2023-04-01T16:33:15.120818Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:15.073857Z",
     "end_time": "2023-04-01T16:33:15.129765Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-04-01T16:33:15.090870Z",
     "end_time": "2023-04-01T16:33:15.130791Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}