calculate_empirical_nevals.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing estimates of running time to experiments\n",
    "\n",
    "\n",
    "The goal here is to create four figures:\n",
    "\n",
    "## Figure 1: average pool size in each step of lexicase\n",
    "\n",
    "- predicted pool size:\n",
    "- actual pool size, averaged over generations and trials\n",
    "- a line representing N, the maximum pool size\n",
    "\n",
    "<!-- ## Figure 2: expected running time comparison\n",
    "\n",
    "boxplots of \n",
    "\n",
    "- sum of all actual lexicase pool sizes per lexicase selection event\n",
    "- sum of expected lexicase pool sizes ($\\bar{L}$ for boolean problems) per selection event (generation)\n",
    "- line at old theory: NT -->\n",
    "\n",
    "## Figure 3: running time as a function of the number of selections\n",
    "\n",
    "line plots of\n",
    "\n",
    "- actual running time ( $ d + \\sum_{i=0}^{d} { S_i} $) per selection event\n",
    "- expected running time ( $ \\hat{d} + \\sum_{i=0}^{\\hat{d}} { \\hat{S_i}} $) per selection event\n",
    "- line of $N*T*s$ where $s$ is the selection event number [1:gens*pop]\n",
    "\n",
    "## Figure 4: measured (actual) running time versus predicted running time\n",
    "\n",
    "scatterplot of\n",
    "\n",
    "- actual running time: $ d + \\sum_{i=0}^{d} { S_i} $\n",
    "- expected running time : $ a + b\\hat{d} + c\\sum_{i=0}^{\\hat{d}} { \\hat{S_i}} $) per selection event\n",
    "    - estimate a,b,c using linear regression\n",
    "    - report R^2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "sns.set_context(\"paper\") #, font_scale = .5, rc={\"grid.linewidth\": 0.6})\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**load data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "from copy import copy\n",
    "from tqdm.notebook import tqdm\n",
    "import numpy as np\n",
    "\n",
    "# what to capture:\n",
    "\n",
    "# data frame of: \n",
    "\n",
    "#   problem, \n",
    "#   trial, \n",
    "#   population size\n",
    "#   training case size\n",
    "#   generation, \n",
    "#   selection_event, \n",
    "#   selection_iteration, \n",
    "#   iteration_pool_size, \n",
    "#   selection_depth, \n",
    "#   average_error\n",
    "dfs = []\n",
    "problems = [\n",
    "    'compare-string-lengths',\n",
    "    'count-odds',\n",
    "    'double-letters',\n",
    "    'last-index-of-zero',\n",
    "    'mirror-image',\n",
    "    'negative-to-zero',\n",
    "    'vector-average',\n",
    "    'x-word-lines'\n",
    "           ]\n",
    "pop_size=1000\n",
    "# fraction of generational selections to include\n",
    "sample_selection = .1\n",
    "\n",
    "for p in tqdm(problems):\n",
    "    for d in tqdm(glob('data/lex-theory-*'+p),p, leave=False):\n",
    "        print(p,'pop_size:',pop_size)\n",
    "    #     frames = []\n",
    "        frame = { \n",
    "            'problem':[],\n",
    "            'trial':[],\n",
    "            'generation':[],\n",
    "            'selection_event':[],\n",
    "#             'n_inds_evaluated':[],\n",
    "            'n_evals':[],\n",
    "#             'selection_depth':[],\n",
    "#             'selection_iteration':[],\n",
    "#             'iteration_pool_size':[],\n",
    "#             'starting_pool_size':[],\n",
    "            'N':pop_size\n",
    "        }\n",
    "        # load individuals remaining\n",
    "        print('\\tindividuals remaining...')\n",
    "        for f in tqdm(glob(d + '/individuals_*.csv'),d, leave=False):\n",
    "            trial = int(f.split('individuals_remaining')[-1].split('.csv')[0])\n",
    "            sel_col_start = -1\n",
    "            gen_counter = {}\n",
    "            with open(f) as fp:\n",
    "                for cnt, line in enumerate(fp):\n",
    "                    vals  = line.split(',')\n",
    "                    if cnt == 0:\n",
    "                        sel_col_start = [i for i,v in enumerate(vals) if v == 'S0'][0]\n",
    "                        continue\n",
    "                    # only capture first sample_selection selection events per generation\n",
    "                    if vals[0] in gen_counter.keys():\n",
    "                        gen_counter[vals[0]] += 1\n",
    "                    else:\n",
    "                        gen_counter[vals[0]] = 1\n",
    "                    if gen_counter[vals[0]] > sample_selection*pop_size: \n",
    "                        continue\n",
    "#                     for i, v in enumerate(vals[sel_col_start:]):\n",
    "#                         v = v.strip()\n",
    "                    frame['problem'].append(p)\n",
    "                    frame['trial'].append(trial)\n",
    "                    frame['generation'].append(vals[0])\n",
    "                    frame['selection_event'].append( cnt)\n",
    "#                     nie = np.sum(vals[sel_col_start:])\n",
    "#                     frame['n_inds_evaluated'].append(nie) # nie!!\n",
    "                    nir = [int(v) for v in vals[sel_col_start:]]\n",
    "                    frame['n_evals'].append(np.sum(nir))\n",
    "#                     frame['selection_iteration'].append( i)\n",
    "#                     frame['iteration_pool_size'].append( int(v))\n",
    "#                     frame['starting_pool_size'].append( int(vals[sel_col_start]))\n",
    "    #                     frames.append(frame)\n",
    "        dfs.append(pd.DataFrame.from_records(frame))\n",
    "df = pd.concat(dfs)\n",
    "del dfs\n",
    "df.to_parquet('data/empirical_num_evals.parquet')\n",
    "print('len(df):',len(df))\n",
    "display(df)\n",
    "del df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('done preprocessing. clearing data.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}