run_3_ecos18-hdf-linear_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading database ...\n",
      "Time to load database: 2.8837778568267822\n",
      "['rpm']\n",
      "['frp']\n",
      "['exh_T']\n",
      "['TC_rpm']\n",
      "['rpm', 'frp']\n",
      "['rpm', 'exh_T']\n",
      "['rpm', 'TC_rpm']\n",
      "['frp', 'exh_T']\n",
      "['frp', 'TC_rpm']\n",
      "['exh_T', 'TC_rpm']\n",
      "['rpm', 'frp', 'exh_T']\n",
      "['rpm', 'frp', 'TC_rpm']\n",
      "['rpm', 'exh_T', 'TC_rpm']\n",
      "['frp', 'exh_T', 'TC_rpm']\n",
      "['rpm', 'frp', 'exh_T', 'TC_rpm']\n",
      "\n",
      "Number of combinations: 15\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import sklearn as sk\n",
    "import time\n",
    "import numpy as np\n",
    "from tpot import TPOTRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.externals import joblib\n",
    "import matplotlib.pyplot as plt\n",
    "#%pylab inline\n",
    "#%%\n",
    "\n",
    "t1 = time.time()\n",
    "print('Loading database ...')\n",
    "df = pd.read_hdf('database/all_data_comp.h5','table')\n",
    "print('Time to load database:', time.time()-t1)\n",
    "#%%\n",
    "\n",
    "# Variable names.\n",
    "import var_names\n",
    "d = var_names.d\n",
    "\n",
    "# Check number of combinations, just to be sure.\n",
    "\n",
    "\n",
    "features =  ['rpm',\n",
    "             'frp',\n",
    "             'exh_T',\n",
    "             'TC_rpm']\n",
    "\n",
    "\n",
    "import itertools\n",
    "\n",
    "def list_of_combs(arr):\n",
    "    \"\"\"returns a list of all subsets of a list\"\"\"\n",
    "    \n",
    "    combs = []\n",
    "    for i in range(1, len(arr)+1):\n",
    "        listing = [list(x) for x in itertools.combinations(arr, i)]\n",
    "        combs.extend(listing)\n",
    "    return combs\n",
    "\n",
    "# Not used, does not produce a good list..\n",
    "#\n",
    "#for l in range(1, len(features)+1):\n",
    "#    for subset in itertools.combinations(features, l):\n",
    "#        print(subset)\n",
    "#        \n",
    "#comb = list()\n",
    "#\n",
    "#for i in range(0,len(features)):\n",
    "#    for a in itertools.combinations(features,i+1):\n",
    "#        comb.append(a)\n",
    "\n",
    "combinations = list_of_combs(features)\n",
    "for i in range(len(combinations)):\n",
    "    print(combinations[i])\n",
    "\n",
    "print('\\nNumber of combinations:',len(combinations))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test: 1 ae1_rpm\n",
      "Test: 1 ae3_rpm\n",
      "Test: 1 me1_rpm\n",
      "Test: 1 me3_rpm\n",
      "Test: 2 ae1_frp\n",
      "Test: 2 ae3_frp\n",
      "Test: 2 me1_frp\n",
      "Test: 2 me3_frp\n",
      "Test: 3 ae1_exh_T\n",
      "Test: 3 ae3_exh_T\n",
      "Test: 3 me1_exh_T\n",
      "Test: 3 me3_exh_T\n",
      "Test: 4 ae1_TC_rpm\n",
      "Test: 4 ae3_TC_rpm\n",
      "Test: 4 me1_TC_rpm\n",
      "Test: 4 me3_TC_rpm\n",
      "Test: 5 ae1_rpm\n",
      "Test: 5 ae3_rpm\n",
      "Test: 5 me1_rpm\n",
      "Test: 5 me3_rpm\n",
      "Test: 5 ae1_frp\n",
      "Test: 5 ae3_frp\n",
      "Test: 5 me1_frp\n",
      "Test: 5 me3_frp\n",
      "Test: 6 ae1_rpm\n",
      "Test: 6 ae3_rpm\n",
      "Test: 6 me1_rpm\n",
      "Test: 6 me3_rpm\n",
      "Test: 6 ae1_exh_T\n",
      "Test: 6 ae3_exh_T\n",
      "Test: 6 me1_exh_T\n",
      "Test: 6 me3_exh_T\n",
      "Test: 7 ae1_rpm\n",
      "Test: 7 ae3_rpm\n",
      "Test: 7 me1_rpm\n",
      "Test: 7 me3_rpm\n",
      "Test: 7 ae1_TC_rpm\n",
      "Test: 7 ae3_TC_rpm\n",
      "Test: 7 me1_TC_rpm\n",
      "Test: 7 me3_TC_rpm\n",
      "Test: 8 ae1_frp\n",
      "Test: 8 ae3_frp\n",
      "Test: 8 me1_frp\n",
      "Test: 8 me3_frp\n",
      "Test: 8 ae1_exh_T\n",
      "Test: 8 ae3_exh_T\n",
      "Test: 8 me1_exh_T\n",
      "Test: 8 me3_exh_T\n",
      "Test: 9 ae1_frp\n",
      "Test: 9 ae3_frp\n",
      "Test: 9 me1_frp\n",
      "Test: 9 me3_frp\n",
      "Test: 9 ae1_TC_rpm\n",
      "Test: 9 ae3_TC_rpm\n",
      "Test: 9 me1_TC_rpm\n",
      "Test: 9 me3_TC_rpm\n",
      "Test: 10 ae1_exh_T\n",
      "Test: 10 ae3_exh_T\n",
      "Test: 10 me1_exh_T\n",
      "Test: 10 me3_exh_T\n",
      "Test: 10 ae1_TC_rpm\n",
      "Test: 10 ae3_TC_rpm\n",
      "Test: 10 me1_TC_rpm\n",
      "Test: 10 me3_TC_rpm\n",
      "Test: 11 ae1_rpm\n",
      "Test: 11 ae3_rpm\n",
      "Test: 11 me1_rpm\n",
      "Test: 11 me3_rpm\n",
      "Test: 11 ae1_frp\n",
      "Test: 11 ae3_frp\n",
      "Test: 11 me1_frp\n",
      "Test: 11 me3_frp\n",
      "Test: 11 ae1_exh_T\n",
      "Test: 11 ae3_exh_T\n",
      "Test: 11 me1_exh_T\n",
      "Test: 11 me3_exh_T\n",
      "Test: 12 ae1_rpm\n",
      "Test: 12 ae3_rpm\n",
      "Test: 12 me1_rpm\n",
      "Test: 12 me3_rpm\n",
      "Test: 12 ae1_frp\n",
      "Test: 12 ae3_frp\n",
      "Test: 12 me1_frp\n",
      "Test: 12 me3_frp\n",
      "Test: 12 ae1_TC_rpm\n",
      "Test: 12 ae3_TC_rpm\n",
      "Test: 12 me1_TC_rpm\n",
      "Test: 12 me3_TC_rpm\n",
      "Test: 13 ae1_rpm\n",
      "Test: 13 ae3_rpm\n",
      "Test: 13 me1_rpm\n",
      "Test: 13 me3_rpm\n",
      "Test: 13 ae1_exh_T\n",
      "Test: 13 ae3_exh_T\n",
      "Test: 13 me1_exh_T\n",
      "Test: 13 me3_exh_T\n",
      "Test: 13 ae1_TC_rpm\n",
      "Test: 13 ae3_TC_rpm\n",
      "Test: 13 me1_TC_rpm\n",
      "Test: 13 me3_TC_rpm\n",
      "Test: 14 ae1_frp\n",
      "Test: 14 ae3_frp\n",
      "Test: 14 me1_frp\n",
      "Test: 14 me3_frp\n",
      "Test: 14 ae1_exh_T\n",
      "Test: 14 ae3_exh_T\n",
      "Test: 14 me1_exh_T\n",
      "Test: 14 me3_exh_T\n",
      "Test: 14 ae1_TC_rpm\n",
      "Test: 14 ae3_TC_rpm\n",
      "Test: 14 me1_TC_rpm\n",
      "Test: 14 me3_TC_rpm\n",
      "Test: 15 ae1_rpm\n",
      "Test: 15 ae3_rpm\n",
      "Test: 15 me1_rpm\n",
      "Test: 15 me3_rpm\n",
      "Test: 15 ae1_frp\n",
      "Test: 15 ae3_frp\n",
      "Test: 15 me1_frp\n",
      "Test: 15 me3_frp\n",
      "Test: 15 ae1_exh_T\n",
      "Test: 15 ae3_exh_T\n",
      "Test: 15 me1_exh_T\n",
      "Test: 15 me3_exh_T\n",
      "Test: 15 ae1_TC_rpm\n",
      "Test: 15 ae3_TC_rpm\n",
      "Test: 15 me1_TC_rpm\n",
      "Test: 15 me3_TC_rpm\n",
      "Test: 16 ae2_rpm\n",
      "Test: 16 ae4_rpm\n",
      "Test: 16 me2_rpm\n",
      "Test: 16 me4_rpm\n",
      "Test: 17 ae2_frp\n",
      "Test: 17 ae4_frp\n",
      "Test: 17 me2_frp\n",
      "Test: 17 me4_frp\n",
      "Test: 18 ae2_exh_T\n",
      "Test: 18 ae4_exh_T\n",
      "Test: 18 me2_exh_T\n",
      "Test: 18 me4_exh_T\n",
      "Test: 19 ae2_TC_rpm\n",
      "Test: 19 ae4_TC_rpm\n",
      "Test: 19 me2_TC_rpm\n",
      "Test: 19 me4_TC_rpm\n",
      "Test: 20 ae2_rpm\n",
      "Test: 20 ae4_rpm\n",
      "Test: 20 me2_rpm\n",
      "Test: 20 me4_rpm\n",
      "Test: 20 ae2_frp\n",
      "Test: 20 ae4_frp\n",
      "Test: 20 me2_frp\n",
      "Test: 20 me4_frp\n",
      "Test: 21 ae2_rpm\n",
      "Test: 21 ae4_rpm\n",
      "Test: 21 me2_rpm\n",
      "Test: 21 me4_rpm\n",
      "Test: 21 ae2_exh_T\n",
      "Test: 21 ae4_exh_T\n",
      "Test: 21 me2_exh_T\n",
      "Test: 21 me4_exh_T\n",
      "Test: 22 ae2_rpm\n",
      "Test: 22 ae4_rpm\n",
      "Test: 22 me2_rpm\n",
      "Test: 22 me4_rpm\n",
      "Test: 22 ae2_TC_rpm\n",
      "Test: 22 ae4_TC_rpm\n",
      "Test: 22 me2_TC_rpm\n",
      "Test: 22 me4_TC_rpm\n",
      "Test: 23 ae2_frp\n",
      "Test: 23 ae4_frp\n",
      "Test: 23 me2_frp\n",
      "Test: 23 me4_frp\n",
      "Test: 23 ae2_exh_T\n",
      "Test: 23 ae4_exh_T\n",
      "Test: 23 me2_exh_T\n",
      "Test: 23 me4_exh_T\n",
      "Test: 24 ae2_frp\n",
      "Test: 24 ae4_frp\n",
      "Test: 24 me2_frp\n",
      "Test: 24 me4_frp\n",
      "Test: 24 ae2_TC_rpm\n",
      "Test: 24 ae4_TC_rpm\n",
      "Test: 24 me2_TC_rpm\n",
      "Test: 24 me4_TC_rpm\n",
      "Test: 25 ae2_exh_T\n",
      "Test: 25 ae4_exh_T\n",
      "Test: 25 me2_exh_T\n",
      "Test: 25 me4_exh_T\n",
      "Test: 25 ae2_TC_rpm\n",
      "Test: 25 ae4_TC_rpm\n",
      "Test: 25 me2_TC_rpm\n",
      "Test: 25 me4_TC_rpm\n",
      "Test: 26 ae2_rpm\n",
      "Test: 26 ae4_rpm\n",
      "Test: 26 me2_rpm\n",
      "Test: 26 me4_rpm\n",
      "Test: 26 ae2_frp\n",
      "Test: 26 ae4_frp\n",
      "Test: 26 me2_frp\n",
      "Test: 26 me4_frp\n",
      "Test: 26 ae2_exh_T\n",
      "Test: 26 ae4_exh_T\n",
      "Test: 26 me2_exh_T\n",
      "Test: 26 me4_exh_T\n",
      "Test: 27 ae2_rpm\n",
      "Test: 27 ae4_rpm\n",
      "Test: 27 me2_rpm\n",
      "Test: 27 me4_rpm\n",
      "Test: 27 ae2_frp\n",
      "Test: 27 ae4_frp\n",
      "Test: 27 me2_frp\n",
      "Test: 27 me4_frp\n",
      "Test: 27 ae2_TC_rpm\n",
      "Test: 27 ae4_TC_rpm\n",
      "Test: 27 me2_TC_rpm\n",
      "Test: 27 me4_TC_rpm\n",
      "Test: 28 ae2_rpm\n",
      "Test: 28 ae4_rpm\n",
      "Test: 28 me2_rpm\n",
      "Test: 28 me4_rpm\n",
      "Test: 28 ae2_exh_T\n",
      "Test: 28 ae4_exh_T\n",
      "Test: 28 me2_exh_T\n",
      "Test: 28 me4_exh_T\n",
      "Test: 28 ae2_TC_rpm\n",
      "Test: 28 ae4_TC_rpm\n",
      "Test: 28 me2_TC_rpm\n",
      "Test: 28 me4_TC_rpm\n",
      "Test: 29 ae2_frp\n",
      "Test: 29 ae4_frp\n",
      "Test: 29 me2_frp\n",
      "Test: 29 me4_frp\n",
      "Test: 29 ae2_exh_T\n",
      "Test: 29 ae4_exh_T\n",
      "Test: 29 me2_exh_T\n",
      "Test: 29 me4_exh_T\n",
      "Test: 29 ae2_TC_rpm\n",
      "Test: 29 ae4_TC_rpm\n",
      "Test: 29 me2_TC_rpm\n",
      "Test: 29 me4_TC_rpm\n",
      "Test: 30 ae2_rpm\n",
      "Test: 30 ae4_rpm\n",
      "Test: 30 me2_rpm\n",
      "Test: 30 me4_rpm\n",
      "Test: 30 ae2_frp\n",
      "Test: 30 ae4_frp\n",
      "Test: 30 me2_frp\n",
      "Test: 30 me4_frp\n",
      "Test: 30 ae2_exh_T\n",
      "Test: 30 ae4_exh_T\n",
      "Test: 30 me2_exh_T\n",
      "Test: 30 me4_exh_T\n",
      "Test: 30 ae2_TC_rpm\n",
      "Test: 30 ae4_TC_rpm\n",
      "Test: 30 me2_TC_rpm\n",
      "Test: 30 me4_TC_rpm\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# The combinations are done manually in an Excel workbook, it was too tedious to make. This way it is easier but\n",
    "# might not be the best way... it works...\n",
    "\n",
    "\n",
    "feat = pd.read_excel('training_setup.xlsx',index_col='test_no')\n",
    "\n",
    "# Create a list of features for each test. A list which will contain a list of features for each row. This list\n",
    "# will be used for the training.\n",
    "\n",
    "test_features = list()\n",
    "\n",
    "\n",
    "for i in range(30):\n",
    "    \n",
    "    tmp_l = list()\n",
    "    \n",
    "    if feat.iloc[i][0] == 1:\n",
    "        #print('ett')\n",
    "        for j in range(1,5):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        \n",
    "    elif feat.iloc[i][0] == 2:\n",
    "        #print('två')\n",
    "        for j in range(1,5):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        for j in range(5,9):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        \n",
    "    elif feat.iloc[i][0] == 3:\n",
    "        for j in range(1,5):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        for j in range(5,9):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        for j in range(9,13):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        \n",
    "    elif feat.iloc[i][0] == 4:\n",
    "        for j in range(1,5):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        for j in range(5,9):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        for j in range(9,13):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "        for j in range(13,17):\n",
    "            tmp_l.append(d[feat.iloc[i][j]])\n",
    "            print('Test:',i+1,feat.iloc[i][j])\n",
    "    \n",
    "    test_features.append(tmp_l)\n",
    "\n",
    "# And then at last add the corresponding predictor to each test set.\n",
    "\n",
    "\n",
    "for i in range(len(test_features)):\n",
    "    if 'AE1' in test_features[i][0]:\n",
    "        test_features[i].append(d['fo_booster_13'])\n",
    "    if 'AE2' in test_features[i][0]:\n",
    "        test_features[i].append(d['fo_booster_24'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       AE1 ENG SPEED:1745:RPM:Average:900  \\\n",
      "count                        30624.000000   \n",
      "mean                           394.698595   \n",
      "std                            367.883957   \n",
      "min                              0.000000   \n",
      "25%                              0.000000   \n",
      "50%                            737.733333   \n",
      "75%                            750.466667   \n",
      "max                            760.000000   \n",
      "\n",
      "       AE1 EXH MEAN VALUE:1591:  C:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                222.944978   \n",
      "std                                 146.610908   \n",
      "min                                   0.000000   \n",
      "25%                                  75.466667   \n",
      "50%                                 320.400000   \n",
      "75%                                 359.533333   \n",
      "max                                 431.066667   \n",
      "\n",
      "       AE1 FUEL RACK POSIT:1742:mm:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                 10.828951   \n",
      "std                                  10.937992   \n",
      "min                                   0.000000   \n",
      "25%                                   0.000000   \n",
      "50%                                  12.733333   \n",
      "75%                                  17.600000   \n",
      "max                                  48.533333   \n",
      "\n",
      "       AE1 TC SPEED:1748:RPM:Average:900  AE2 ENG SPEED:2745:RPM:Average:900  \\\n",
      "count                       30624.000000                        30624.000000   \n",
      "mean                         8349.694446                          324.487335   \n",
      "std                          8203.742140                          365.052774   \n",
      "min                             0.000000                            0.000000   \n",
      "25%                             0.000000                            0.000000   \n",
      "50%                         10692.600000                            0.000000   \n",
      "75%                         15090.150000                          750.266667   \n",
      "max                         24527.066667                          754.333333   \n",
      "\n",
      "       AE2 EXH MEAN VALUE:2591:  C:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                203.659239   \n",
      "std                                 140.633203   \n",
      "min                                   0.000000   \n",
      "25%                                  81.000000   \n",
      "50%                                  86.933333   \n",
      "75%                                 357.400000   \n",
      "max                                 429.400000   \n",
      "\n",
      "       AE2 FUEL RACK POSIT:2742:mm:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                 12.654099   \n",
      "std                                  10.278427   \n",
      "min                                   0.000000   \n",
      "25%                                   4.000000   \n",
      "50%                                   5.000000   \n",
      "75%                                  20.333333   \n",
      "max                                  43.000000   \n",
      "\n",
      "       AE2 TC SPEED:2748:RPM:Average:900  AE3 ENG SPEED:3745:RPM:Average:900  \\\n",
      "count                       30624.000000                        30624.000000   \n",
      "mean                         6759.406479                          169.412385   \n",
      "std                          7990.940027                          307.666435   \n",
      "min                             0.000000                            0.000000   \n",
      "25%                             0.000000                            0.000000   \n",
      "50%                             0.000000                            0.000000   \n",
      "75%                         14241.416667                            0.000000   \n",
      "max                         23800.533333                          756.200000   \n",
      "\n",
      "       AE3 EXH MEAN VALUE:3591:  C:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                146.880291   \n",
      "std                                 130.109081   \n",
      "min                                   0.000000   \n",
      "25%                                  72.000000   \n",
      "50%                                  79.800000   \n",
      "75%                                  88.483333   \n",
      "max                                 454.333333   \n",
      "\n",
      "                      ...                   \\\n",
      "count                 ...                    \n",
      "mean                  ...                    \n",
      "std                   ...                    \n",
      "min                   ...                    \n",
      "25%                   ...                    \n",
      "50%                   ...                    \n",
      "75%                   ...                    \n",
      "max                   ...                    \n",
      "\n",
      "       ME2 FUEL RACK POSIT:20005:%:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                 12.165520   \n",
      "std                                  19.433878   \n",
      "min                                   0.000000   \n",
      "25%                                   0.000000   \n",
      "50%                                   0.000000   \n",
      "75%                                  25.330285   \n",
      "max                                  89.889918   \n",
      "\n",
      "       ME2 T/C SPEED:20006:rpm:Average:900  \\\n",
      "count                         30624.000000   \n",
      "mean                           4484.403686   \n",
      "std                            6732.386500   \n",
      "min                               0.000000   \n",
      "25%                              36.630035   \n",
      "50%                              36.630035   \n",
      "75%                           10825.152653   \n",
      "max                           21827.594661   \n",
      "\n",
      "       ME3 ENGINE SPEED:3364:rpm:Average:900  \\\n",
      "count                           30624.000000   \n",
      "mean                              136.519933   \n",
      "std                               185.477938   \n",
      "min                                 0.000000   \n",
      "25%                                 0.328042   \n",
      "50%                                 0.730159   \n",
      "75%                               375.550278   \n",
      "max                               502.962980   \n",
      "\n",
      "       ME3 EXH GAS MEAN:3125:C:Average:900  \\\n",
      "count                         30624.000000   \n",
      "mean                            187.907736   \n",
      "std                             156.340571   \n",
      "min                               0.000000   \n",
      "25%                              70.294671   \n",
      "50%                              78.607248   \n",
      "75%                             395.918007   \n",
      "max                             469.385443   \n",
      "\n",
      "       ME3 FUEL RACK POSIT:30005:%:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                 12.547920   \n",
      "std                                  19.230252   \n",
      "min                                   0.000000   \n",
      "25%                                   0.000000   \n",
      "50%                                   0.000000   \n",
      "75%                                  26.783032   \n",
      "max                                  89.212679   \n",
      "\n",
      "       ME3 T/C SPEED:30006:rpm:Average:900  \\\n",
      "count                         30624.000000   \n",
      "mean                           4757.749316   \n",
      "std                            6797.442550   \n",
      "min                               0.000000   \n",
      "25%                              21.978022   \n",
      "50%                              29.304029   \n",
      "75%                           11484.004980   \n",
      "max                           21405.616927   \n",
      "\n",
      "       ME4 ENGINE SPEED:4364:rpm:Average:900  \\\n",
      "count                           30624.000000   \n",
      "mean                              124.419687   \n",
      "std                               180.295213   \n",
      "min                                 0.000000   \n",
      "25%                                 0.000000   \n",
      "50%                                 0.000000   \n",
      "75%                               367.100541   \n",
      "max                               498.359806   \n",
      "\n",
      "       ME4 EXH GAS MEAN:4125:C:Average:900  \\\n",
      "count                         30624.000000   \n",
      "mean                            180.895840   \n",
      "std                             151.565511   \n",
      "min                               0.000000   \n",
      "25%                              72.834964   \n",
      "50%                              81.544569   \n",
      "75%                             393.849229   \n",
      "max                             482.653668   \n",
      "\n",
      "       ME4 FUEL RACK POSIT:40005:%:Average:900  \\\n",
      "count                             30624.000000   \n",
      "mean                                 11.743487   \n",
      "std                                  19.084399   \n",
      "min                                   0.000000   \n",
      "25%                                   0.000000   \n",
      "50%                                   0.000000   \n",
      "75%                                  23.711781   \n",
      "max                                  89.494467   \n",
      "\n",
      "       ME4 T/C SPEED:40006:rpm:Average:900  \n",
      "count                         30624.000000  \n",
      "mean                           4594.686134  \n",
      "std                            6881.782836  \n",
      "min                               0.000000  \n",
      "25%                              80.586082  \n",
      "50%                              80.586082  \n",
      "75%                           10869.963289  \n",
      "max                           21875.946484  \n",
      "\n",
      "[8 rows x 34 columns]\n"
     ]
    }
   ],
   "source": [
    "df_train = pd.DataFrame()\n",
    "for i in range(len(test_features)):\n",
    "    df_train = df_train.append(df[test_features[i]]).resample('15min').mean()\n",
    "    \n",
    "\n",
    "df_train = df_train.dropna()\n",
    "\n",
    "# Filter out below zero\n",
    "    \n",
    "\n",
    "for n in list(df_train):\n",
    "    df_train[n][(df_train[n] < 0)] = 0\n",
    "    \n",
    "\n",
    "date_begin = '2014-02-01'\n",
    "date_end = '2014-12-16'\n",
    "\n",
    "df_train = df_train[date_begin:date_end]\n",
    "\n",
    "print(df_train.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test run 0 \n",
      "\n",
      "Test number, linear model:  0 \n",
      "\n",
      "MSE:  0.020751914179531383\n",
      "MSE %:  0.007660924530644874\n",
      "R2 score: 0.8449911584110031\n",
      "['ae1_rpm: 0.0002829647723174473', 'ae3_rpm: -3.7667819035563184e-05', 'me1_rpm: 0.0015379412879452927', 'me3_rpm: 0.0014808872294868962']\n",
      "Test run 1 \n",
      "\n",
      "Test number, linear model:  1 \n",
      "\n",
      "MSE:  0.008844792204968992\n",
      "MSE %:  0.0032652065243377777\n",
      "R2 score: 0.9389967717585165\n",
      "['ae1_frp: 0.012281406154461333', 'ae3_frp: -0.0009344122536376592', 'me1_frp: 0.01306719102837232', 'me3_frp: 0.01301745863697864']\n",
      "Test run 2 \n",
      "\n",
      "Test number, linear model:  2 \n",
      "\n",
      "MSE:  0.02505056265261032\n",
      "MSE %:  0.009247844236033382\n",
      "R2 score: 0.8067858465965971\n",
      "['ae1_exh_T: 0.0007453837697878059', 'ae3_exh_T: -4.613598962062903e-05', 'me1_exh_T: 0.0017192736986634044', 'me3_exh_T: 0.0017620229308720264']\n",
      "Test run 3 \n",
      "\n",
      "Test number, linear model:  3 \n",
      "\n",
      "MSE:  0.005299930998506011\n",
      "MSE %:  0.0019565602982893966\n",
      "R2 score: 0.964493994771804\n",
      "['ae1_TC_rpm: 1.80703531208048e-05', 'ae3_TC_rpm: 2.819436730238895e-06', 'me1_TC_rpm: 4.1419536595205265e-05', 'me3_TC_rpm: 4.125753576407307e-05']\n",
      "Test run 4 \n",
      "\n",
      "Test number, linear model:  4 \n",
      "\n",
      "MSE:  0.004761374730946821\n",
      "MSE %:  0.0017577430284422488\n",
      "R2 score: 0.9680686377785997\n",
      "['ae1_rpm: 0.0002408190164663561', 'ae3_rpm: 7.310067625757049e-05', 'me1_rpm: 0.00030670636185559357', 'me3_rpm: 0.0002282978142928502', 'ae1_frp: 0.0057612255295729185', 'ae3_frp: -0.0014032939866814153', 'me1_frp: 0.010773083208463688', 'me3_frp: 0.011491940991374527']\n",
      "Test run 5 \n",
      "\n",
      "Test number, linear model:  5 \n",
      "\n",
      "MSE:  0.013496726123207937\n",
      "MSE %:  0.004982547602411783\n",
      "R2 score: 0.9028687588738891\n",
      "['ae1_rpm: -0.0006332372644556141', 'ae3_rpm: -0.0008462483040174173', 'me1_rpm: 0.003547939851223054', 'me3_rpm: 0.0031496921673519807', 'ae1_exh_T: 0.002431042779061982', 'ae3_exh_T: 0.0019392955695466173', 'me1_exh_T: -0.002328487839175561', 'me3_exh_T: -0.0020282967103825845']\n",
      "Test run 6 \n",
      "\n",
      "Test number, linear model:  6 \n",
      "\n",
      "MSE:  0.004192350558380518\n",
      "MSE %:  0.0015476780096478461\n",
      "R2 score: 0.9721497628094518\n",
      "['ae1_rpm: -0.00029741109754755714', 'ae3_rpm: 0.00018827140158596953', 'me1_rpm: -0.00019006235107685445', 'me3_rpm: -0.00015662054208984052', 'ae1_TC_rpm: 3.0947773992108984e-05', 'ae3_TC_rpm: -4.778490342737917e-06', 'me1_TC_rpm: 4.642543483687811e-05', 'me3_TC_rpm: 4.520927638401618e-05']\n",
      "Test run 7 \n",
      "\n",
      "Test number, linear model:  7 \n",
      "\n",
      "MSE:  0.004278515209266746\n",
      "MSE %:  0.0015794871662363908\n",
      "R2 score: 0.971383717863834\n",
      "['ae1_frp: 0.0049606475638741', 'ae3_frp: -0.003639773087759543', 'me1_frp: 0.010983665548883324', 'me3_frp: 0.01159407058709582', 'ae1_exh_T: 0.0006621208665534076', 'ae3_exh_T: 0.00037165765766768844', 'me1_exh_T: 0.0003234263249896242', 'me3_exh_T: 0.0002657340743967523']\n",
      "Test run 8 \n",
      "\n",
      "Test number, linear model:  8 \n",
      "\n",
      "MSE:  0.00294087746126671\n",
      "MSE %:  0.001085675282276419\n",
      "R2 score: 0.9805544972502882\n",
      "['ae1_frp: 0.0006376717088031544', 'ae3_frp: -0.005079021382927182', 'me1_frp: 0.007273571086187486', 'me3_frp: 0.008687433363712267', 'ae1_TC_rpm: 1.7979301614829553e-05', 'ae3_TC_rpm: 9.63706447120219e-06', 'me1_TC_rpm: 1.8619302592502608e-05', 'me3_TC_rpm: 1.469779425528925e-05']\n",
      "Test run 9 \n",
      "\n",
      "Test number, linear model:  9 \n",
      "\n",
      "MSE:  0.003846514895842582\n",
      "MSE %:  0.0014200068518072865\n",
      "R2 score: 0.9745088629024975\n",
      "['ae1_exh_T: -0.000587056251200566', 'ae3_exh_T: 0.0008638832896736161', 'me1_exh_T: -0.0005739474664971752', 'me3_exh_T: -0.0005023314751136142', 'ae1_TC_rpm: 2.9016804797058e-05', 'ae3_TC_rpm: -1.2382226227548157e-05', 'me1_TC_rpm: 5.321523160243132e-05', 'me3_TC_rpm: 5.0668531192769645e-05']\n",
      "Test run 10 \n",
      "\n",
      "Test number, linear model:  10 \n",
      "\n",
      "MSE:  0.004120157070409706\n",
      "MSE %:  0.001521026547129081\n",
      "R2 score: 0.9724349788196551\n",
      "['ae1_rpm: -0.0001609980817675867', 'ae3_rpm: -0.0002727014619773328', 'me1_rpm: 0.00034920172881132286', 'me3_rpm: 0.00019169787402745675', 'ae1_frp: 0.00499743739781466', 'ae3_frp: -0.005396725402332292', 'me1_frp: 0.010250995785238004', 'me3_frp: 0.010947749043109377', 'ae1_exh_T: 0.0010558419093700542', 'ae3_exh_T: 0.0011539393758827766', 'me1_exh_T: 1.5284034078954026e-05', 'me3_exh_T: 0.00011511380504828977']\n",
      "Test run 11 \n",
      "\n",
      "Test number, linear model:  11 \n",
      "\n",
      "MSE:  0.002465145045259375\n",
      "MSE %:  0.0009100505131932692\n",
      "R2 score: 0.9837632235093704\n",
      "['ae1_rpm: -0.00023339760043875876', 'ae3_rpm: 0.00011675560054849505', 'me1_rpm: 0.00017181571803082509', 'me3_rpm: 0.00025450036124493344', 'ae1_frp: -0.00059671992210409', 'ae3_frp: -0.0039834072057238585', 'me1_frp: 0.006546840469962961', 'me3_frp: 0.009187769306587035', 'ae1_TC_rpm: 2.9046977841300847e-05', 'ae3_TC_rpm: 3.3329164163429113e-06', 'me1_TC_rpm: 1.697603057391264e-05', 'me3_TC_rpm: 7.194726709735776e-06']\n",
      "Test run 12 \n",
      "\n",
      "Test number, linear model:  12 \n",
      "\n",
      "MSE:  0.003098685907044607\n",
      "MSE %:  0.001143932972769149\n",
      "R2 score: 0.979502500700276\n",
      "['ae1_rpm: -0.00023139274383900287', 'ae3_rpm: -0.00046358889783683057', 'me1_rpm: 0.0012866601575178783', 'me3_rpm: 0.0008450001782405024', 'ae1_exh_T: -2.914880813949182e-05', 'ae3_exh_T: 0.002350656307023249', 'me1_exh_T: -0.001615537226930287', 'me3_exh_T: -0.0011716182110873981', 'ae1_TC_rpm: 2.9075610709648266e-05', 'ae3_TC_rpm: -2.0032995504054564e-05', 'me1_TC_rpm: 4.324900175328856e-05', 'me3_TC_rpm: 4.256178767951627e-05']\n",
      "Test run 13 \n",
      "\n",
      "Test number, linear model:  13 \n",
      "\n",
      "MSE:  0.0024779036455080555\n",
      "MSE %:  0.0009147605689874578\n",
      "R2 score: 0.983694339075126\n",
      "['ae1_frp: -0.0009722549313904077', 'ae3_frp: -0.0031334817898170525', 'me1_frp: 0.006699778174914682', 'me3_frp: 0.009891803941876567', 'ae1_exh_T: -0.0006472884191260569', 'ae3_exh_T: 0.0005885967689476792', 'me1_exh_T: 5.927249862803445e-05', 'me3_exh_T: 0.00020860711060792286', 'ae1_TC_rpm: 3.0815870809659166e-05', 'ae3_TC_rpm: -3.3982779288705073e-06', 'me1_TC_rpm: 1.933880762271889e-05', 'me3_TC_rpm: 6.9276066283396664e-06']\n",
      "Test run 14 \n",
      "\n",
      "Test number, linear model:  14 \n",
      "\n",
      "MSE:  0.0023109893512973305\n",
      "MSE %:  0.000853141298592851\n",
      "R2 score: 0.984789501939991\n",
      "['ae1_rpm: -0.00015147420359655262', 'ae3_rpm: -0.0002995885174744479', 'me1_rpm: 0.0005842755648894923', 'me3_rpm: 0.00032677420454871266', 'ae1_frp: -0.0008376178270807664', 'ae3_frp: -0.003545040428126006', 'me1_frp: 0.005241287413385747', 'me3_frp: 0.008666048173313904', 'ae1_exh_T: -0.000260550729283984', 'ae3_exh_T: 0.0016102219116570973', 'me1_exh_T: -0.0005346610237279617', 'me3_exh_T: -0.000126324595998144', 'ae1_TC_rpm: 3.039729341799839e-05', 'ae3_TC_rpm: -8.640244578281586e-06', 'me1_TC_rpm: 2.166911971590215e-05', 'me3_TC_rpm: 8.881225509733485e-06']\n",
      "Test run 15 \n",
      "\n",
      "Test number, linear model:  15 \n",
      "\n",
      "MSE:  0.021935940856046058\n",
      "MSE %:  0.008072145172806718\n",
      "R2 score: 0.8479180774422101\n",
      "['ae2_rpm: 0.0002786850777765881', 'ae4_rpm: 0.00023031794673124956', 'me2_rpm: 0.0015253675815115233', 'me4_rpm: 0.0016058548643990336']\n",
      "Test run 16 \n",
      "\n",
      "Test number, linear model:  16 \n",
      "\n",
      "MSE:  0.004023515231423077\n",
      "MSE %:  0.0014806020524118202\n",
      "R2 score: 0.9748512061779129\n",
      "['ae2_frp: 0.014793441386068563', 'ae4_frp: 0.015096369035471353', 'me2_frp: 0.012891086166914378', 'me4_frp: 0.01364055911194231']\n",
      "Test run 17 \n",
      "\n",
      "Test number, linear model:  17 \n",
      "\n",
      "MSE:  0.024263503488967084\n",
      "MSE %:  0.008928658398979107\n",
      "R2 score: 0.830045101083306\n",
      "['ae2_exh_T: 0.000836739237246768', 'ae4_exh_T: 0.0007009431499204355', 'me2_exh_T: 0.001716477874586949', 'me4_exh_T: 0.001847380210533512']\n",
      "Test run 18 \n",
      "\n",
      "Test number, linear model:  18 \n",
      "\n",
      "MSE:  0.0036932029202371122\n",
      "MSE %:  0.0013590513541419753\n",
      "R2 score: 0.977282781785587\n",
      "['ae2_TC_rpm: 1.8600655576318314e-05', 'ae4_TC_rpm: 1.741115566448276e-05', 'me2_TC_rpm: 4.065181407171457e-05', 'me4_TC_rpm: 4.0353000328458186e-05']\n",
      "Test run 19 \n",
      "\n",
      "Test number, linear model:  19 \n",
      "\n",
      "MSE:  0.0019082993054793844\n",
      "MSE %:  0.0007022296936376952\n",
      "R2 score: 0.9882710538628899\n",
      "['ae2_rpm: 0.00018451628928938628', 'ae4_rpm: 1.5332977327633693e-05', 'me2_rpm: 0.0002535882530760522', 'me4_rpm: 0.00032221286063093967', 'ae2_frp: 0.00853177757139716', 'ae4_frp: 0.01407581264814673', 'me2_frp: 0.011112393535494243', 'me4_frp: 0.011153541967035229']\n",
      "Test run 20 \n",
      "\n",
      "Test number, linear model:  20 \n",
      "\n",
      "MSE:  0.009947120245445001\n",
      "MSE %:  0.0036604127992287947\n",
      "R2 score: 0.9358324645794343\n",
      "['ae2_rpm: -0.001355240420368781', 'ae4_rpm: -0.001522320592791669', 'me2_rpm: 0.0029048393950660894', 'me4_rpm: 0.003746754333312026', 'ae2_exh_T: 0.0044360214568603335', 'ae4_exh_T: 0.004857716346158976', 'me2_exh_T: -0.0017301802232885905', 'me4_exh_T: -0.0026514054518100194']\n",
      "Test run 21 \n",
      "\n",
      "Test number, linear model:  21 \n",
      "\n",
      "MSE:  0.002652170188890649\n",
      "MSE %:  0.0009759646476168718\n",
      "R2 score: 0.9837548074839354\n",
      "['ae2_rpm: -0.00017152446738460616', 'ae4_rpm: -0.00018099575196533807', 'me2_rpm: -0.0002507222379583554', 'me4_rpm: -0.0001783313361346248', 'ae2_TC_rpm: 2.6704344244678635e-05', 'ae4_TC_rpm: 2.62072287077794e-05', 'me2_TC_rpm: 4.708095665585716e-05', 'me4_TC_rpm: 4.46497649939257e-05']\n",
      "Test run 22 \n",
      "\n",
      "Test number, linear model:  22 \n",
      "\n",
      "MSE:  0.001599725118142246\n",
      "MSE %:  0.0005886783464166026\n",
      "R2 score: 0.9901917108228506\n",
      "['ae2_frp: 0.006008480249136239', 'ae4_frp: 0.01300308065268525', 'me2_frp: 0.011562074269911795', 'me4_frp: 0.0116914320374193', 'ae2_exh_T: 0.0006690773373405931', 'ae4_exh_T: 0.00012768871702636595', 'me2_exh_T: 0.00022832264145908793', 'me4_exh_T: 0.0003007958466247949']\n",
      "Test run 23 \n",
      "\n",
      "Test number, linear model:  23 \n",
      "\n",
      "MSE:  0.0008845369123642278\n",
      "MSE %:  0.0003254982502992243\n",
      "R2 score: 0.9946107308714283\n",
      "['ae2_frp: 0.0010280443049100166', 'ae4_frp: 0.005299189503611625', 'me2_frp: 0.009112668686829644', 'me4_frp: 0.008985333473134429', 'ae2_TC_rpm: 1.8392791026358747e-05', 'ae4_TC_rpm: 1.2400797716945502e-05', 'me2_TC_rpm: 1.2444801233888873e-05', 'me4_TC_rpm: 1.3792353172061015e-05']\n",
      "Test run 24 \n",
      "\n",
      "Test number, linear model:  24 \n",
      "\n",
      "MSE:  0.0022135384883186157\n",
      "MSE %:  0.0008145538019345157\n",
      "R2 score: 0.9864573040645489\n",
      "['ae2_exh_T: -0.0004609833930486956', 'ae4_exh_T: -0.0004616559064690404', 'me2_exh_T: -0.000553953137888645', 'me4_exh_T: -0.0005800279208965249', 'ae2_TC_rpm: 2.763657372239576e-05', 'ae4_TC_rpm: 2.6740451052175473e-05', 'me2_TC_rpm: 5.229769122369468e-05', 'me4_TC_rpm: 5.163398000796066e-05']\n",
      "Test run 25 \n",
      "\n",
      "Test number, linear model:  25 \n",
      "\n",
      "MSE:  0.0010632392517284225\n",
      "MSE %:  0.0003912584214965478\n",
      "R2 score: 0.9934991127548841\n",
      "['ae2_rpm: -0.000650438163457418', 'ae4_rpm: -0.00018584927329309527', 'me2_rpm: 0.00044668109001339625', 'me4_rpm: 0.0006250447789280195', 'ae2_frp: 0.0023037045647569935', 'ae4_frp: 0.010608222784984443', 'me2_frp: 0.010549004573628881', 'me4_frp: 0.010593906256666643', 'ae2_exh_T: 0.002588027701918945', 'ae4_exh_T: 0.0007748378896175303', 'me2_exh_T: -0.0001776034835416693', 'me4_exh_T: -0.00031427861428293076']\n",
      "Test run 26 \n",
      "\n",
      "Test number, linear model:  26 \n",
      "\n",
      "MSE:  0.0006060160063739812\n",
      "MSE %:  0.00022300612554519307\n",
      "R2 score: 0.9963126209314614\n",
      "['ae2_rpm: -0.00013603972801012968', 'ae4_rpm: -0.00013505332899789023', 'me2_rpm: 0.00021135119036705418', 'me4_rpm: 0.0002801760303404878', 'ae2_frp: -0.000147633517035158', 'ae4_frp: 0.0012670419907122909', 'me2_frp: 0.009256205676718908', 'me4_frp: 0.009538696771006', 'ae2_TC_rpm: 2.5561214116191182e-05', 'ae4_TC_rpm: 2.308652943116507e-05', 'me2_TC_rpm: 6.898740784219877e-06', 'me4_TC_rpm: 5.7271545173846775e-06']\n",
      "Test run 27 \n",
      "\n",
      "Test number, linear model:  27 \n",
      "\n",
      "MSE:  0.0014608773016471998\n",
      "MSE %:  0.0005375841289845601\n",
      "R2 score: 0.9910714723546719\n",
      "['ae2_rpm: -0.00026672726769350486', 'ae4_rpm: -0.000563694660566526', 'me2_rpm: 0.0008740820520686924', 'me4_rpm: 0.0014015917275667113', 'ae2_exh_T: 0.0005778008196857773', 'ae4_exh_T: 0.0016560309925659892', 'me2_exh_T: -0.0013381480099314222', 'me4_exh_T: -0.0019489916442201162', 'ae2_TC_rpm: 2.125703525565265e-05', 'ae4_TC_rpm: 1.5515730166709657e-05', 'me2_TC_rpm: 4.61477685349065e-05', 'me4_TC_rpm: 4.468793310360231e-05']\n",
      "Test run 28 \n",
      "\n",
      "Test number, linear model:  28 \n",
      "\n",
      "MSE:  0.0006664556381563791\n",
      "MSE %:  0.00024524713563635716\n",
      "R2 score: 0.9959445753700399\n",
      "['ae2_frp: 7.593115368904283e-06', 'ae4_frp: 0.0014714561390980047', 'me2_frp: 0.009635837445429774', 'me4_frp: 0.009802576809672552', 'ae2_exh_T: -0.0005095610228606092', 'ae4_exh_T: -0.0004942653844327825', 'me2_exh_T: 0.0001919002782371235', 'me4_exh_T: 0.00024825309875810053', 'ae2_TC_rpm: 2.8301191082319217e-05', 'ae4_TC_rpm: 2.5411497286494377e-05', 'me2_TC_rpm: 7.159084790810038e-06', 'me4_TC_rpm: 6.88074250611843e-06']\n",
      "Test run 29 \n",
      "\n",
      "Test number, linear model:  29 \n",
      "\n",
      "MSE:  0.0006033488893543416\n",
      "MSE %:  0.00022202466065537226\n",
      "R2 score: 0.9963282010502729\n",
      "['ae2_rpm: -0.00015669752277443867', 'ae4_rpm: -0.00016900077550651322', 'me2_rpm: 0.0003574105158316161', 'me4_rpm: 0.0005993328676441496', 'ae2_frp: -0.00011659692475382013', 'ae4_frp: 0.0013789408017171626', 'me2_frp: 0.008443712133027647', 'me4_frp: 0.008159339571762812', 'ae2_exh_T: 0.00010248772421506494', 'ae4_exh_T: 0.00014596425522143356', 'me2_exh_T: -0.00022335758373042466', 'me4_exh_T: -0.0004713682907483024', 'ae2_TC_rpm: 2.4707903791842207e-05', 'ae4_TC_rpm: 2.2023547082505876e-05', 'me2_TC_rpm: 1.03231129751943e-05', 'me4_TC_rpm: 1.1308751141694967e-05']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from sklearn.externals import joblib\n",
    "from tpot import TPOTRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import linear_model\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "gen = 10\n",
    "cores = -1\n",
    "\n",
    "results = list()\n",
    "cols = ['test_no','model','MSE','MSE_perc','R2-score','coef']\n",
    "\n",
    "df_out = pd.DataFrame()\n",
    "\n",
    "df_out['y_13'] = df_train[d['fo_booster_13']]\n",
    "df_out['y_24'] = df_train[d['fo_booster_24']]\n",
    "\n",
    "\n",
    "seed = 42 # This is to get reproduce.\n",
    "\n",
    "\n",
    "for test_no in range(len(test_features)):\n",
    "    \n",
    "    # Feature selection for the test run\n",
    "    print('Test run', test_no, '\\n')\n",
    "    \n",
    "    df_test = df_train[test_features[test_no]]\n",
    "    # drop the last column, which is the measured\n",
    "    X = np.array(df_test.drop(df_test.columns[len(df_test.columns)-1],axis=1))\n",
    "    # y is the measured value\n",
    "    y = np.array(df_test[df_test.columns[len(df_test.columns)-1]])\n",
    "    \n",
    "    # make a test and train split.\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, test_size=0.25, random_state=seed)\n",
    "    \n",
    "    # \n",
    "    \n",
    "    print('Test number, linear model: ', test_no, '\\n')\n",
    "    m_linear = sk.linear_model.LinearRegression(n_jobs=cores)\n",
    "    m_linear.fit(X_train, y_train.reshape(-1,))\n",
    "    \n",
    "    df_out['linear_test_no_'+str(test_no)] = m_linear.predict(X)\n",
    "    \n",
    "    MSE = mean_squared_error(m_linear.predict(X_test),y_test)\n",
    "    MSE_perc = mean_squared_error(m_linear.predict(X_test),y_test)/max(y_test)\n",
    "    R_2 = r2_score(m_linear.predict(X_test),y_test)\n",
    "    \n",
    "    readable = list()\n",
    "    for g in range(len(list(df_test))-1):\n",
    "        #print(g)\n",
    "        readable.append(d[list(df_test)[g]]+': '+str(m_linear.coef_[g]))\n",
    "    \n",
    "    print('MSE: ',MSE )\n",
    "    print('MSE %: ',MSE_perc )\n",
    "    print('R2 score:', R_2)\n",
    "    print(readable)\n",
    "    \n",
    "    results.append([test_no,'linear',MSE,MSE_perc,R_2,readable])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'copy_X': True, 'fit_intercept': True, 'n_jobs': -1, 'normalize': False}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_linear.get_params(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results, columns=cols)\n",
    "results.to_excel('results/run_180210/linear_run.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0002829647723174473"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_linear.coef_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_selection import RFE\n",
    "selector = RFE(m_linear, 5, step=1)\n",
    "selector = selector.fit(X, y)\n",
    "selector.support_\n",
    "selector.ranking_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2.82964772e-04, -3.76678190e-05,  1.53794129e-03,  1.48088723e-03])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_linear.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test run 0 \n",
      "\n",
      "Test number, linear model:  0 \n",
      "\n",
      "MSE:  0.0207519141795314\n",
      "MSE %:  0.00766092453064488\n",
      "R2 score: 0.8449911584110014\n",
      "Test number, TPOT model:  0 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.004130317639937286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.0041301529063429146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.0040646702020535035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.004053847319654523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.004053847319654523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.004053847319654523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.004053847319654523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.004053847319654523\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.0040536438569537744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.004053223901289258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: ExtraTreesRegressor(MaxAbsScaler(input_matrix), bootstrap=True, max_features=0.8500000000000001, min_samples_leaf=1, min_samples_split=20, n_estimators=100)\n",
      "MSE:  0.004019807487257575\n",
      "MSE %:  0.0014839807798538661\n",
      "R2 score: 0.9731131789295406\n"
     ]
    }
   ],
   "source": [
    "# Train models and produce results...\n",
    "# \n",
    "\n",
    "from sklearn.externals import joblib\n",
    "from tpot import TPOTRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import linear_model\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "### Train a linear model just for comparison.\n",
    "\n",
    "gen = 10\n",
    "cores = -1\n",
    "\n",
    "results = list()\n",
    "cols = ['test_no','model','MSE','MSE_perc','R2-score']\n",
    "\n",
    "df_out = pd.DataFrame()\n",
    "\n",
    "df_out['y_13'] = df_train[d['fo_booster_13']]\n",
    "df_out['y_24'] = df_train[d['fo_booster_24']]\n",
    "\n",
    "\n",
    "seed = 42 # This is to get reproduce.\n",
    "\n",
    "\n",
    "#for test_no in range(len(test_features)):\n",
    "for test_no in range(0,1):\n",
    "    \n",
    "    # Feature selection for the test run\n",
    "    print('Test run', test_no, '\\n')\n",
    "    \n",
    "    df_test = df_train[test_features[test_no]]\n",
    "    # drop the last column, which is the measured\n",
    "    X = np.array(df_test.drop(df_test.columns[len(df_test.columns)-1],axis=1))\n",
    "    # y is the measured value\n",
    "    y = np.array(df_test[df_test.columns[len(df_test.columns)-1]])\n",
    "    \n",
    "    # make a test and train split.\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, test_size=0.25, random_state=seed)\n",
    "    \n",
    "    # \n",
    "    \n",
    "    print('Test number, linear model: ', test_no, '\\n')\n",
    "    m_linear = sk.linear_model.LinearRegression(n_jobs=cores)\n",
    "    m_linear.fit(X_train, y_train.reshape(-1,))\n",
    "    \n",
    "    df_out['linear_test_no_'+str(test_no)] = m_linear.predict(X)\n",
    "    \n",
    "    MSE = mean_squared_error(m_linear.predict(X_test),y_test)\n",
    "    MSE_perc = mean_squared_error(m_linear.predict(X_test),y_test)/max(y_test)\n",
    "    R_2 = r2_score(m_linear.predict(X_test),y_test)\n",
    "    \n",
    "    print('MSE: ',MSE )\n",
    "    print('MSE %: ',MSE_perc )\n",
    "    print('R2 score:', R_2)\n",
    "    \n",
    "    results.append([test_no,'linear',MSE,MSE_perc,R_2])\n",
    "    \n",
    "    # tpot model\n",
    "    \n",
    "    print('Test number, TPOT model: ', test_no, '\\n')\n",
    "\n",
    "    m_tpot = TPOTRegressor(generations=gen, population_size=50, verbosity=2, n_jobs=cores)\n",
    "    m_tpot.fit(X_train, y_train.reshape(-1,))\n",
    "    \n",
    "    df_out['tpot_test_no_'+str(test_no)] = m_tpot.predict(X)\n",
    "    \n",
    "    MSE = mean_squared_error(m_tpot.predict(X_test),y_test)\n",
    "    MSE_perc = mean_squared_error(m_tpot.predict(X_test),y_test)/max(y_test)\n",
    "    R_2 = r2_score(m_tpot.predict(X_test),y_test)\n",
    "    \n",
    "    print('MSE: ',MSE )\n",
    "    print('MSE %: ',MSE_perc )\n",
    "    print('R2 score:', R_2)\n",
    "    \n",
    "    results.append([test_no,'tpot',MSE,MSE_perc,R_2])\n",
    "    \n",
    "\n",
    "    \n",
    "\n",
    "results = pd.DataFrame(results, columns=cols)\n",
    "results.to_excel('results/run_180210/test_run.xlsx')\n",
    "df_out.to_hdf('results/run_180210/df_out.h5','table',complevel=9,complib='blosc')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test run 0 \n",
      "\n",
      "Test number, linear model:  0 \n",
      "\n",
      "MSE:  0.0207519141795314\n",
      "MSE %:  0.00766092453064488\n",
      "R2 score: 0.8449911584110014\n",
      "Test number, TPOT model:  0 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.004040208327555095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.004018545793392216\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.003992184172375004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.003992184172375004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: XGBRegressor(input_matrix, learning_rate=0.1, max_depth=7, min_child_weight=9, n_estimators=100, nthread=1, subsample=0.9500000000000001)\n",
      "MSE:  0.004031966109240564\n",
      "MSE %:  0.0014884693433956427\n",
      "R2 score: 0.9731434643639371\n",
      "Test run 1 \n",
      "\n",
      "Test number, linear model:  1 \n",
      "\n",
      "MSE:  0.00884479220496899\n",
      "MSE %:  0.003265206524337777\n",
      "R2 score: 0.938996771758517\n",
      "Test number, TPOT model:  1 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.0018826913144613737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.0018707748815437373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.001855723305107387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.0018389154185768752\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.0017995438730053631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.0017995438730053631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.0017995438730053631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.0017995438730053631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.0017995438730053631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.0017995438730053631\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: XGBRegressor(LassoLarsCV(input_matrix, normalize=True), learning_rate=0.1, max_depth=7, min_child_weight=17, n_estimators=100, nthread=1, subsample=1.0)\n",
      "MSE:  0.001946974253497957\n",
      "MSE %:  0.000718758890872269\n",
      "R2 score: 0.9872125498430397\n",
      "Test run 2 \n",
      "\n",
      "Test number, linear model:  2 \n",
      "\n",
      "MSE:  0.02505056265261032\n",
      "MSE %:  0.009247844236033382\n",
      "R2 score: 0.8067858465965964\n",
      "Test number, TPOT model:  2 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.005330280125418293\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.005330280125418293\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.005255888704972102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.005255888704972102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.005198894053834885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.005198894053834885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.00518801914677282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.00518801914677282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.00518801914677282\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.005184058377506879\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: XGBRegressor(DecisionTreeRegressor(ExtraTreesRegressor(input_matrix, bootstrap=False, max_features=0.7500000000000001, min_samples_leaf=1, min_samples_split=6, n_estimators=100), max_depth=5, min_samples_leaf=18, min_samples_split=13), learning_rate=0.1, max_depth=7, min_child_weight=4, n_estimators=100, nthread=1, subsample=0.9500000000000001)\n",
      "MSE:  0.0048022979725040925\n",
      "MSE %:  0.0017728505439421362\n",
      "R2 score: 0.9682018495050114\n",
      "Test run 3 \n",
      "\n",
      "Test number, linear model:  3 \n",
      "\n",
      "MSE:  0.005299930998506014\n",
      "MSE %:  0.001956560298289398\n",
      "R2 score: 0.9644939947718036\n",
      "Test number, TPOT model:  3 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.0016932033353033952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.0016932033353033952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.0016309734156011286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: ExtraTreesRegressor(input_matrix, bootstrap=False, max_features=0.55, min_samples_leaf=1, min_samples_split=15, n_estimators=100)\n",
      "MSE:  0.001719326307163226\n",
      "MSE %:  0.0006347187526306193\n",
      "R2 score: 0.9886275476512637\n",
      "Test run 4 \n",
      "\n",
      "Test number, linear model:  4 \n",
      "\n",
      "MSE:  0.004761374730946834\n",
      "MSE %:  0.0017577430284422535\n",
      "R2 score: 0.9680686377785986\n",
      "Test number, TPOT model:  4 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.001183142425751077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.001183142425751077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.001183142425751077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.0011662673063863884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.0011534897671329144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.001148921992750118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.001148921992750118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.001148921992750118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.0011338147037312834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.0011338147037312834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: RandomForestRegressor(input_matrix, bootstrap=False, max_features=0.4, min_samples_leaf=2, min_samples_split=4, n_estimators=100)\n",
      "MSE:  0.0012814821194305577\n",
      "MSE %:  0.0004730810719144\n",
      "R2 score: 0.9915940430849012\n",
      "Test run 5 \n",
      "\n",
      "Test number, linear model:  5 \n",
      "\n",
      "MSE:  0.013496726123207993\n",
      "MSE %:  0.004982547602411803\n",
      "R2 score: 0.902868758873885\n",
      "Test number, TPOT model:  5 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.0010375640840909121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.0010375640840909121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.0010375640840909121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.0009768134646414232\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.0009387502430655327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 6 - Current best internal CV score: -0.0009387502430655327\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 7 - Current best internal CV score: -0.0009352551784988719\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 8 - Current best internal CV score: -0.0008910858493256623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 9 - Current best internal CV score: -0.0008756124060192123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 10 - Current best internal CV score: -0.0008754541141005572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Best pipeline: ExtraTreesRegressor(ElasticNetCV(PolynomialFeatures(input_matrix, degree=2, include_bias=False, interaction_only=False), l1_ratio=0.7000000000000001, tol=0.0001), bootstrap=False, max_features=0.6500000000000001, min_samples_leaf=1, min_samples_split=3, n_estimators=100)\n",
      "MSE:  0.000817125008278709\n",
      "MSE %:  0.0003016556914397917\n",
      "R2 score: 0.9946590027626764\n",
      "Test run 6 \n",
      "\n",
      "Test number, linear model:  6 \n",
      "\n",
      "MSE:  0.004192350558380513\n",
      "MSE %:  0.0015476780096478442\n",
      "R2 score: 0.9721497628094513\n",
      "Test number, TPOT model:  6 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 1 - Current best internal CV score: -0.0013597315254979567\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 2 - Current best internal CV score: -0.0013597315254979567\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 3 - Current best internal CV score: -0.0013004442028298022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 4 - Current best internal CV score: -0.0013004442028298022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 5 - Current best internal CV score: -0.0013004442028298022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Optimization Progress:  55%|███████████████████████████                      | 304/550 [20:09<3:21:32, 49.16s/pipeline]"
     ]
    }
   ],
   "source": [
    "# Train models and produce results...\n",
    "# \n",
    "\n",
    "from sklearn.externals import joblib\n",
    "from tpot import TPOTRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import linear_model\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "### Train a linear model just for comparison.\n",
    "\n",
    "gen = 10\n",
    "cores = -1\n",
    "\n",
    "results = list()\n",
    "cols = ['test_no','model','MSE','MSE_perc','R2-score']\n",
    "\n",
    "df_out = pd.DataFrame()\n",
    "\n",
    "df_out['y_13'] = df_train[d['fo_booster_13']]\n",
    "df_out['y_24'] = df_train[d['fo_booster_24']]\n",
    "\n",
    "\n",
    "seed = 42 # This is to get reproduce.\n",
    "\n",
    "\n",
    "for test_no in range(len(test_features)):\n",
    "    \n",
    "    # Feature selection for the test run\n",
    "    print('Test run', test_no, '\\n')\n",
    "    \n",
    "    df_test = df_train[test_features[test_no]]\n",
    "    # drop the last column, which is the measured\n",
    "    X = np.array(df_test.drop(df_test.columns[len(df_test.columns)-1],axis=1))\n",
    "    # y is the measured value\n",
    "    y = np.array(df_test[df_test.columns[len(df_test.columns)-1]])\n",
    "    \n",
    "    # make a test and train split.\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.75, test_size=0.25, random_state=seed)\n",
    "    \n",
    "    # \n",
    "    \n",
    "    print('Test number, linear model: ', test_no, '\\n')\n",
    "    m_linear = sk.linear_model.LinearRegression(n_jobs=cores)\n",
    "    m_linear.fit(X_train, y_train.reshape(-1,))\n",
    "    \n",
    "    df_out['linear_test_no_'+str(test_no)] = m_linear.predict(X)\n",
    "    \n",
    "    MSE = mean_squared_error(m_linear.predict(X_test),y_test)\n",
    "    MSE_perc = mean_squared_error(m_linear.predict(X_test),y_test)/max(y_test)\n",
    "    R_2 = r2_score(m_linear.predict(X_test),y_test)\n",
    "    \n",
    "    print('MSE: ',MSE )\n",
    "    print('MSE %: ',MSE_perc )\n",
    "    print('R2 score:', R_2)\n",
    "    \n",
    "    results.append([test_no,'linear',MSE,MSE_perc,R_2])\n",
    "    \n",
    "    # tpot model\n",
    "    \n",
    "    print('Test number, TPOT model: ', test_no, '\\n')\n",
    "\n",
    "    m_tpot = TPOTRegressor(generations=gen, population_size=50, verbosity=2, n_jobs=cores)\n",
    "    m_tpot.fit(X_train, y_train.reshape(-1,))\n",
    "    \n",
    "    df_out['tpot_test_no_'+str(test_no)] = m_tpot.predict(X)\n",
    "    \n",
    "    MSE = mean_squared_error(m_tpot.predict(X_test),y_test)\n",
    "    MSE_perc = mean_squared_error(m_tpot.predict(X_test),y_test)/max(y_test)\n",
    "    R_2 = r2_score(m_tpot.predict(X_test),y_test)\n",
    "    \n",
    "    print('MSE: ',MSE )\n",
    "    print('MSE %: ',MSE_perc )\n",
    "    print('R2 score:', R_2)\n",
    "    \n",
    "    results.append([test_no,'tpot',MSE,MSE_perc,R_2])\n",
    "    \n",
    "\n",
    "results = pd.DataFrame(results, columns=cols)\n",
    "results.to_excel('results/run_180210/2_test_run.xlsx')\n",
    "df_out.to_hdf('results/run_180110/2_df_out.h5','table',complevel=9,complib='blosc')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "module 'matplotlib.pyplot' has no attribute 'labels'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-91-6b2d842cb7be>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'tpot_test_no_0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_out\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'linear_test_no_0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdate\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: module 'matplotlib.pyplot' has no attribute 'labels'"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzsnXecXGW9/9/Pmd53tu9mN9l0kpAAIUAgoVoookgVC1wVfl5FRcHGVS568SrYRcTCVRSRjoh0KQGkCglppNdNtvednT5z5vn9cc7s7mRn++zOZnPerxcvds+cNpuZz/k+3yqklBgYGBgYTC+UfN+AgYGBgUHuMcTdwMDAYBpiiLuBgYHBNMQQdwMDA4NpiCHuBgYGBtMQQ9wNDAwMpiGGuBsYGBhMQwxxNzAwMJiGGOJuYGBgMA0x5+vCxcXFsqamJl+XNzAwMDgsWbduXZuUsmS4/fIm7jU1NaxduzZflzcwMDA4LBFC1I5kP8MtY2BgYDANMcTdwMDAYBpiiLuBgYHBNMQQdwMDA4NpiCHuBgYGBtMQQ9wNDAwMpiGGuBsYGBhMQ/KW525gMBa2NgR4aUcLJkXgtJpw28ycubAUv8ua71szMJhSGOJuMCVJpSQ9sSTheJJwXGVzXTd/fauWtbWdA/b12M184Yy5fHbVbOwWUx7u1sBg6mGIu8GU4c097by4rZlN9d1sbQgQjCUzXp9d7OLGDy3i4uVV2C0mQvEkdZ0Rbn9xFz9+dgd/eaOWT62cyekLSllS6UVRRJ7eiYFB/hFSyrxceMWKFdJoP2CQJqGmOPZ/niOZkiyq8LJ0ho9ZRU5cNjNOq4kKn4MVs/yDCvZbe9v52XM7eGe/ZtkXuax87rQ5/OfpcyfzbRgYTDhCiHVSyhXD7WdY7gZTgq0NAUJxlV9/4jjOX1Y56uNXzini4c+fQmtPjNd2t/Kbl/Zw39sHDHE3OGIZNltGCHGXEKJFCPHeIK8LIcSvhBC7hRCbhBDLc3+bBtOdtC99xazCcZ2nxGPjwuOqOG1BCS2BGPlamRoY5JuRpEL+GThniNfPBebr/30O+O34b8vgSGPt/g6qCx2U++w5OV+px0YkoQ7w2xsYHCkMK+5Syn8BHUPscgHwF6nxFlAghKjI1Q0aTH+klLyzv5MTxmm196fUawOgpSeWs3MaGBxO5KKIaQZwsN/vdfo2A4MRUdsepi0YY0VNDsXdo60AWgKGuBscmeRC3LOlL2R1dAohPieEWCuEWNva2pqDSxtMB97Zry0MT6jx5+ycpZ605R7N2TkNDA4nciHudUB1v9+rgIZsO0op75RSrpBSrigpGXZKlMERwtr9nRQ4LcwtcefsnGnLvdVwyxgcoeRC3B8HrtSzZlYC3VLKxhyc1+AI4Z3aDo6fOXgO+1jwOsxYzYrhczc4Yhk2z10IcT9wBlAshKgDvgtYAKSUvwOeBs4DdgNh4DMTdbMG04/2YIy9rSEuPb56+J1HgRCCUo+NloDhljE4MhlW3KWUHx/mdQl8MWd3ZHBEsU7Pb8+lvz1NqcdmWO4GRyxGy1+DvLK2thOrWWFplS/n5y712A1xNzhiMcT9CENKOaUySN7Z38ExVT5s5tx3cyz1Gm4ZgyMXQ9yPMJ7f2syqW9fQPAVEL5pQea++O6f57f0p9dgIRJNEE+qEnN/AYCpjiPsRxsa6LhKqZH9bKN+3Ql1bNz9VbmeFo3lCzm+kQxocyRjifoSxp0UT9aYpYLnHWnZzgekNFtfeMyHnL/EahUwGRy6GuB9h7G0LAtDYnX/Bi3VrFntJ3T8hmXvrurdK1WhBYHAEYoj7EURSTbG/LQxA0xQQ90SP1oLCHA/A7hdzfv7e/jKGW8ZgtOx9GZ76er7vYlwY4n4EUdcZIa6mAGjsjuT5bkANtgMgTTZ475Gcn7/IZcWkCMMtYzB6Xv0ZvPN/kDp8g/GGuB9B7GnVXDJeu3lKWO4yrIv70ktgxzMQz22QV1EExW6r4ZYxGB3BVtj/mvZzrCe/9zIODHE/gtjbqonnyXOLpoTP3RRuJ4gD5dhPQCKsCXyOMQqZDEbNtsdBaitc4sH83ss4MMT9CGJPa5Bit5Wjyr20BmMkdBdNvjDHOgkIL8w8BTyV8N7fcn4NowWBwajZ+ljfz4blbnA4sKc1yJxiNxU+O1LmP9Boi3cSVLygKHD0RbDreYh05vQapV4brYbP3WCkpF0yFcdqv8cMy93gMGBva4i5pa7eOaVNeQ6qOpJdhCx6w7CjL4ZUIufWe4nHTnsoTjLPqxSDw4TtT2gumeM+pf0eNyx3gylOZyhOeyjO3BI3FT4HkP9cd4/aTcxSoP1SeRxULodnv53pe1eT8PKP4J6LIDD6MQGlHhtSQlswnqO7NpjWbHkMiubBzJXa74ZbxmCqky5emlPS33LPr7h7ZYCEXbfchYBP/Q3KlsADn4SND0LHPvjTufDyD2Hfv+CuD0Lb7lFdwxi3ZzBiQm2w/1VY/FGw6lPBDLeMwVQn3XZgbokbr92M02rKq+Uu4yEcxEjZ+zUNcxbCfzwONavg75+D366C1h1w8R/hqucgHtYEvn7diK9T6jUGZRuMkG26S2bJR8Hm1bYZlrvBVGdPWxCrSaHK70QIQbnPnlfLPdLdpv3gLM58weaBTzwMSy/VlsZfeB2WXgIzlmsCb3XBnz8M7/wRUsP70fssd0PcDYZh13Pgr4Gyo8GmW+6Gz91gqrOnJURNsROTPqe03GvPa5VqsFPrK6O4iga+aLHDxX+AKx6Fgn7j94rmwlXPQ9Xx8NT18OcPQduuIa9T7DbcMgYjJNQK/tmai9BsA5PVcMsYTH32tgaZW+Lu/T3vlnuXJu4Wb8noDvSUw5WPw0d+DS1bNNfNmv8d+CXsrIXNj2B96zZ+4vgzy3b+WgvOGhgMRqxHWzmmsboPa7fMsDNUDQ5/EmqKAx1hzlta0butwmenuSeGmpK91vxkEgtoTcNsoxV30Cyr5VfA/A/CP78N//oJrP8rvO8mUCyw/i9aAFbnHOHG0xKE16rg9G/m6i0YTDeiAbB7+363uY0KVYOpTW17mGRKMqfE1but3OdATUnagvnxRas9rSSBx7teoDE4+hRHADxlcMkf4bPPgacCHvsCPHq1ZrWf+R34/OvwX/VcU/Uor9jOgJdvhYNv5/JtjI1wR77vwCAbsZ6+QCpoPx/GbhnDcj8CSDcM6++WqfD2pUOW6T9PJmqonVcdDh45+AAdtHDbWbeN/WQzT4KrX9QCYlYnzFqtVb3qlHrsfL/5Kk737YW/XQ2ffy3TQptMGtbDnWfC51+F8qX5uQeDgaRULXjaX9ytbogF8ndP48Sw3I8A0uKeablrgp6vdEgRbucfbh8Aaw6uYUPLhvGdUFFg4Tkw+7QMYQco89rY02PiF95vkuquI/j3r9IZjBGJq6RScnzXHS21bwIS2keXr28wwaTdL7rPvSscJ5CyGW4Zg6nNwY4wRS4rHruld1vFcC0Inv6m9t8EkYi28ZrTwvlzzqfYUczP1/0cKSdGaC9bUc1HjqnkvsYKbot/FPeOv7Hzx6dxxXd/yYIbn+GlHS0Tct2sNG3W/h9qm7xrGgxPOnBq85BQE1z5+Ne5RB4kfhhb7oZb5gigsTvaa6mnKXRZsZoUGgebpXrgTTBZsr+WAzbQQkwRXDT/Io4rPY7vv/V9Xj74MmfOPDPn16opdnHb5cchpWR38wmse3URS3f9jkeUm1mjHsvufT/lzIWlOb9uVpo2af8PtU7O9QxGRlQT8aDFynUvXsO+2Ftgg/WROCfl+dbGimG5HwE0dUcpP8SvPmwhU6Qr58Mz+vO6NUhBUuH4suO5cP6F1HhruO3d20imJi5dUQjB/HIfx1/6DZxf3wxn/TdnmTZQXffEhF0zg2QMWrdrPxviPrWI9dBiMvHpnX9mbdNaZNtHMaUEr5gO30lMhuV+BNAciLJ8ln/A9nKffXCfe6QTmBg3SXesm7V2yapQAYpQUITCtcuv5fqXr+eaF66h1FmKw+xAEQoxNUZMjSGRuC1uvFYvTouTSDJCOBEmlAiRkimEEAgEJsWEVbFiUbRVRyKVIJ6Kk5IpLIoFm8mG1WRFIMBtQRQUUBGtn5D3OYDW7ZBKEhYCpyHuU4tYD/9VUsTBaDs3nvBTvnZ3lEWONbxij/MNVUWYTPm+w1FjiPs0J5pQ6QwnBljuoPnd1x/oGnhQMq5lDpgm5uPxQu3zJIVgqdpXffr+me/n0gWXsrZ5LfsC+4gkI6RkCrvJ3ivGwUSQnngPqlQRCFwWF06zE0VRkFIikagplUQqQSKVAMCiWLAoFkzCRDwVJ6bGSKiJvrfq97IyspNLJuSdZtJ24A1+UlLEsy4nvw7VceokXNNgZOzv3sPbDjtfmXsxtsQiYD1lwRLWlQXY176VOaWHX2bTiL69QohzgNsAE/AHKeWth7w+E7gbKND3uUFK+XSO79VgDKQbZmUT97RbRkqJEP0KmaK64E+QW+aZPU8wK5Gg1Nwn7kIIbjr5pmGPlVISU2PYTLbMex4jN9yxlH+6umiLtFHsKB52fyklSZnsXRmMBDWl8tDOh7h9x51EXU480syPaGelmsAygXENg5Hz9+Z/Y5KSC+Z8iLvWa/73glAFsIdXDrw4PcVdCGEC7gA+ANQB7wghHpdSbu23243AQ1LK3wohFgNPAzUTcL8Go6RJD5iW6QHVx3Y/xo/f+TEnlp+IVSwjLr10hOIU6T1YgL5pSGpcs+LN1pzdT0u4hbdb1vOfwTDSl6WvzDAIIbCbc5eXf16sgKddXTyw/QG+dNyXBrweTUZ5YPsDPH/geVrDrbRF2jArZn56+k85req0Yc+fkim+8/p3eGrvU6yUNj7bYefpqJ/HZhzk/u33c+WSK3P2XqYqBwIH+P2m3+OxeiiyF1HhruDsmrNH/IBsDjWz5uAaTiw/kbkFc0d9/ZZwCy8eeJEiexEzvTOp9lTjsvSlBSdTSR7vfI9TwxFKCmbzXv1WPHYzqXghC2NxXm54g8/w1aznllKiShWTMOXE2MglI7HcTwR2Syn3AgghHgAuAPqLuwTS2f8+oCGXN2kwdtLiXu6181bjW/zPG/9Dja+Gza2baYm8iHu+mRf3F3PZ0Wf0HdR/1F0ilFNx39CyAYnkjHCEXZWjF/dcU2Iu4cRwBw/ueJCrll6Fw6wNMkmoCR7d9Sh3brqTlkgLy0qWcUL5CRQ5ini9/nVu+NcN3H/+/czyzhr03FJKfvjvH/LU3qf48rFf4qonb+aB6Cl4UwWsCu/kdxt/y/lzz6fQXkggHuC3G36LRHLZwsuY45sDaML0wPYH2Ni6kaXFSzmp4iSOKz0upw+4iURKyc1v3cy7ze9iM9kIJrS88b1de7l2+bXDHv9u87tc9/J1dES1qt4lRUu4YN4FXDz/YqymoT+XiVSC+7fdz282/oZQom8VahImvnfK9/jovI8C8Grdq7SpES4MhpAWJ1sbApw0u5DgDjtnhCP8X+d2uqJdFNgLMs6/o2MH1718HQd7DgKgCIXz55zPD1b/YOR/oAlkJOI+AzjY7/c6GJAd9D3gOSHElwEX8P6c3J3BuGnWA6ZRGrj+peup8dXwl3P/gsvi4u9b3uSmt77FvTv/b4C4X19ajE9V+W48BI6BwdixElM1N5E3lcLkHt4NMtGkHIV8qi3Il10KT+x5gssWXkZzqJlrX7qWre1bWV66nB+d9iNWlK/oPebyhZfzsSc/xlfWfIX7PnQfTosz67lvX387D+54kM8c/Rk+V/0BSHyTrdQgRYpvdnRykcvFHevv4AM1H+DG126kLdKGIhTu3XYvqypX4bf7eXb/s6gplfn++dy95W7++N4fcVlc/PmcP3NU4VGT9WcaMy8eeJF/N/6bb5/0bT5+1MeJqTFufO1G/rL1L1y28DLKXeW9+x4IHGBX1y6q3FVUe6p5cu+T3PLvW6h0V/LzM37O1vatPL7ncX747x+y5sAabjvztkH/9lvatnDj6zeyu2s3q2es5rrjr0NNqRzs0VZM33/z+8z3z2dJ0RIe3f0oRYqNU1ULzT0J2kNxVs4p4rntDs4IR/i938er9a/y4bkf7j3/Swde4luvfguPxcM1x1yDRPLSwZdY1zzyWQMTzUjEPdta49A0io8Df5ZS/kwIcTJwjxDiaCllRsNtIcTngM8BzJw5cyz3azBKmgJR7LYw33r9K1hMFu543x14rFoV3slVy4m3n8pey5NsaNnAsaXaUOA3Wt7leZeTmngi5373tLhbpcTqm6Tc8qFwFnF6rIsFvpO4Z+s9LClawrVrriWYCPLzM37O+2e+f8Byu9JdyU9O/wn/+fx/cuPrN/Kz03+WsU9ntJPfb/o99267l0sWXMJ1y6/j4OsPUg3MX3YK23ZsZ04iyeVV7+e+nQ/z0M6HqPHW8Nfz/kq5q5xHdj7CQzseIpQI8bGFH+OTR32Sam81oUSIdc3r+Na/vsWf3vsTPzrtR5P8xxod0WSUn7zzE+b753PpgksBsJlsXHf8dZo4v3sbt5x6CwA7O3dy5TNXZljYAKtnrOZHp/0Ir9XL8WXHc8XiK/jH7n9w0xs38YUXvsAd77sDt9WdcczTe5/mpjduosBWwC/P/CVnVZ/V+++zqGgRJ5SfwMee/BjXv3Q9v3n/b3i17lWutFRgsUV4r74bgGOrC3jG4mJxPE6xxcPLB1/mw3M/THesmwd3PMiv1/+axUWL+dVZv6LUqX2Oo2qUv27968AYVp4YibjXAf2aalPFQLfLVcA5AFLKN4UQdqAYyCj9k1LeCdwJsGLFikmu+z4yaeyO4JzxN9oibdx19l1Uuit7X/M7rSS6TsBX+TJ3vXcXvzrrV6Rkil/WvwBAk9mEjPVkfbqPlbS426TEPpaOkDnG7ClBAGeXnMvtu3/OJ5/+JKXOUu457x4W+BcMetzKipVct/w6frbuZ1z0+EWcOuNUVlasZG3zWu7ddi+RZITLFlzGt0/6NkII1r/9ChUoXHj2+9ne0A6d8IXy01gbrGVF+Qq+svwrvS6hzx/zea5eejVSyoyAq8vi4rSq07hg3gU8uONBvhH5xoiCwPniT+/9iYZQA3etvguz0ic1le5KrlxyJX/Y/Ac+tehTlDhLuOaFa3CZXfzqzF/REeugrqcOj8XDJQsuwaRkpiFeMO8C7GY7N/zrBq5+7mpuOvkmZrhn4LF6uGPDHdy56U6Wly7nF2f+gsL+k750/HY/Pz/j51z5zJVc8cwVqFLlQlxg87ClIYAQsKjCCzYvSgJO98zl6fpXufKZK9nYupGUTHF2zdl8f9X3e//NAEodpSRSCbpj3QNcOPlgJOL+DjBfCDEbqAcuBz5xyD4HgPcBfxZCLALsgJHIOwXYE3qDhG0LXz/u6ywrWZbxmsNqwm52MM9+Ni8d/Bt7u/ayvWM722JtLIvG2GS30RVuIXdOGYir2qBqmTLj9uT/C2D1albXEmUOc31zcVvd/PLMX45INP9jyX/gtDh5rvY57tl2D3/a8icAzqk5h88f8/ne4N/b+zpwdW6j2zuHIq8Hm7cMOsEXD/PIRx7Jeu7+Yngoly+8nHu33cvDOx/mC8d8YbRveVJoCDbwx/f+yNk1Z3NC+QkDXr/q6Kt4dNej3Pr2rcTUGD3xHu4+9+4Ru5rOrjkbu8nO9S9fz8ee/BgAdpOdqBrlovkXceNJNw6ZiXR08dHccOINfP+t77O8dDmzW7t0ce9mdrELl82MYnNDAj7kms3fOzcRU2NcvfRqTp1xKseUHDPAOi9xasZKS6Tl8BB3KWVSCPEl4J9oaY53SSm3CCFuBtZKKR8Hvgb8nxDiOjSXzaflRDUKMRgxgXiARvMDeEQNn1z0yaz7FDqtlMr3scf0FHduvpMNLRtYaPLw6e5WrreX0Bisz6m4py33kHRT4MpdoHasOAvKAEh0t/Hwhx/GrJizLqmjCZVXdrby7HtNbGno5u7PnkiFz8FlCy/jsoWXEU6EWde8jkp35YCMjld2tnCFUkvBbC0U5fCXQy3IUOuYVkU1vhpWzVjFwzse5uqjr56S6ZS/3/R7BIKvHf+1rK+7rW6+dNyXuPnNmzEJE7efdfuoYwinV5/O4xc+ztb2rTQEG2gINnBU4VF8dN5HR+QWuXTBpQghOKbkGHj4P8FZyJa6QG/Bn9nphSCcYPHx7qfeHbCCOJS0e6Y13Drkqm+yGFGeu56z/vQh227q9/NWYFVub81gvPxy3S9JKT2s8n17UEuwwGklHLFz4VEXcv/2+wH4nW0BBarWtbAx1MziHN5TXI2jSAhID6WO/IuSp0gL6MV72gYVyTd2t3H1X9YSjqs4rSbCcZV3a7v40LK+JbnT4uTUquxlSW1NDZSLDqg8BoAifyFRaUF0N2PLesTwfOKoT/DFF7/ICwde4NzZ547xLBNDMB7kmX3PcN6c86hwVwy634XzLmRDywZWVqwc9G83HDPcM5jhnjGmY4UQvbEAYj3EPdXUd0W44mQtA8pud5LEhDnWM6ywA5Q4dMs9PImN6IbA6C0zTVnfsp6Hdz5MomMVi4oGl+dCl5WOUJz/WPIfmISJkypO4pRYkgq75pZoiuT2gxpX41iBdunBNwXE3aEHdVPBwb2ID609iN1i4t6rT+LNG94HQG3HyAPN5rb3tB/0/u3lBQ7a8BHVRw2OhdUzVlPtqea+bfeN+RwTxbP7nyWSjHDR/IsAiMaTPPXmRn714i7+69HNfPG+d2nsjmBWzPxg9Q8yslCysbslyNV3v0NnKD5xNx3roUPV0kuXVGpZ3V6nlTCOEQ/sSLtlWiNTwyNtiPs05fcbf0+hrZRY6wcGdITsj99lpTOcYIZ7Bn/44B+4ZfUtiGgXfu8MbKkUTdHcTg2KqTGsKUmP4sViyv/HTzi1XHsRbh90n7a9G/mt8/eseufL+B66kAcdt9DePLJ+NFJKCrp3aL/o4l7hs9MuvSR7xv7gVITC5QsvZ0PrBra0bxnzeSaCR3c9yryCeSwr1mI8656/jw/98zTmvnwNGzdv4KlNjby2a+Qtj1/c1swL21r4zctj64GfVFO8tqtt6IHwsQDNMc1NuKRSmzPgtVsI4hhxT3ebyYbP5jMsd4OJpSPaQYVjDkhb5qSl5q1wz0WgW6p+p4UO3SJaUb5Csz4inQhvFRVJlcZYlt4z4yCeimOTkrA5/wEnAMxWgsKFaZCHWENXhEsiD3J86BXoOgixACfJzTib1o7o9G3BOLNSBwlbi8GpZW6U6+IuRtg8bEtDN998ZCPBWGbHzI/O/ygei4f/eeN/iCSHEK5JZGfnTja3beai+Rf1+b2btYfPefb3eEr5Gt80P0BHT3jE59zdoonr3W/UUtc58uPSPL6xgU/98d+cfMsaVv7wxd6VQy9qEhJh6sNmKn12CvVYkNdhpidlR46ip3uJo4TWsGG5G0wgcTWOqmp+wgzLfeezsOdFeF4LmfidVgLRBEm1X0lCpAvcpZSlUjQmcjusIJaIYJcqUcsUEXcgaPJhjXdmfW39ngY+oKwjsOAS+MJrcOXjAFgD+0d07v3tIeYp9cQK5vVuK/XYaceLOTb4aqE/D6+t46G1dVx7/3rUfpOjvFYvt552K9s7tvPd1787YcNORsPfd/0ds2Lm/Dnn925Tgo104UF8eR1i0Ye5xvw4hXUvjviczU31fNf3NFaR5OfP7xz1PW3ftpk77bfzizPtHF/j56lNjTy/tZ9LTBfv+qiFuaV9OfOa5W5HjfaM+FoljhLDLWMwscTUGImkCSGg1NMvbNemfzk23gf7X6fQZUVK6I7onRLVJMS6weGnIqXQpI7eUhqKeCKITUritoH5x/kiYi7Amcgu7j2bn8IlYvhOuFzb4CggYvbhj9cTSw7f63t/a5B5oh5T2cLebVazQthSiCPeCSMQ5HcPdOJzWFizvYX/fWprxmunVZ3Gtcuv5Zn9z3DXe3cNe66JJKbGeGLvE7xv5vvw2/tyrOzhJjpNxeCthHO1wiulZ2RD0aWUHN/2Dz4T+yu3LtjO39fXs61xdAZH2f5/8EHe5MJ3P8PtJwWwmhTqu/pZ7voUpvqwmSp/X5Dc67AQlA5SkVFY7s4Swy1jMLHE1TjxhEKRy5bp227dAVUngm8mPPU1Ch3a0rkzrAerolqFHg4/FcJCayqzRe54icVDWCSoWYpL8kXcVohb7c76WlX903QphZhmr+7dFnFXM5MWDnYM7wppbarFKyK4ZizJ2K7ai7DIeN94t0EIx5NsaQjwqZUzuWr1bP70+n7ueXN/xj5XHX0V59Scw23v3sZr9a8Ne08TxZoDa+iOdfcGUtO4460EbXo1ssOPioISHdmqpTkQ4yS5EYBzux/Ga1P48bPbR3xPTd1RFkY30e2oBm8lyr0X81nX69R3DhT3ppiVGQX9xF233OUIA6qgpUO2R9pJZRbn5wVD3KcpsVSMaEKh3NfPapcS2nZB5bFw3o+hdRtH1/4VgM6wLuDppmGOQiqEFQk0h8ee1THgvhIhbFL2+p+nAqq9kAICAyzx7q4OToivZV/ZB6F/Kpx/NrNEMwdGkDGTbNaEyFR6SA63S6/OHcbvvvFgN2pKcvwsP98+bxHvX1TK957Yyu6WvoeCEIKbV91MibOER3ZmL4qaDJ7Y8wSVrkpWVqzs3SalxK+2E3dq9QQoJkKKF+sIxX1ffRPLxU4i3tmY2ndw69ImXtrRyrra7CutQ3lnbxPHK7tIzH4fXPVPqDmVG+K3U9TyZt9OulumBweV/cXdYSYkHYghAqrheJJEP5dmiaOEpEzSGR3Z/U0khrhPU2LJGJGYyOzjHmjQhnAUL4CF58LCDzHrvdupoL03qNon7n7KTVpTpsbQyJbQIyGejGKTEsWV/46QaaSzCD89dAYzU+3q33oYm0hgOubijO32snnMEG0caM1u7ffH2rFL+6FkYcZ2s14Z239Q9gu//jJr7vtJxn7rajsoo4OTWx7EJFW+cfZRqCnJ9qZMi99hdlDmLCOaHGSy1iTQEm5hgX8BiuiTlZ5QiGLRjfT0tb0IW/w4BnGDHUpw5ytYhUrsA7eAt4oPdD4IwDv7R5bFVb/1TRwijn/xmWD3wWV3A1DU088L/5XpAAAgAElEQVT61y33oDxE3PVsGSU++Orq/Ntf48yfvsxj6+tJpWRfIdMU8Lsb4j4NSQ+0CEVFZqZMm56Slxaa938XJRnhDNOGvhzifuJeYdGCS02hppzdW0yNYZUSm8uTs3OOF5O7GJtI0tmVaU3at/+dOlnMvOMyh3Y7SudiFim6m/YNeV4pJQWhfURNbnCXZZ6jQCueinZrf9twOMSprfezdOevSST6HjLrajv5nucxHGv+G177OUVuLZOjI0vOt9Vk7a0AzgcxNTagFXFrwwEALP6+QqO4rRCP2kUqNXy8wVX3LyJY8R11Jpz8Rcx1b3KWu5ZdzSNzlZgPaha6qUavsbT7iJlcuGMtxJO6xa2Lew/OTLeMQ3PLmJLhrLGRtmCMfa09dIfjfPXBDXz416/R1aMdPxX87oa4T0OSqSQSSTRuyrTcW/VgarEu7kXzkIqFatFKR9rnHtEtIkcB5RYt3zeX4h5NaqmQTpcvZ+ccL+n+MsGOfu8z1M7Mrrf5t/MMnLbMYitRqPVaT7btGfK87aE4s1J1BNxz4JByeE+RVrkZaNdWRbs3vY5NJCihi62vaRk5qZRka20jZ6mvg9kBL9+Kv309QmgplodiM9l6e/fkg6gaxWbKrLkNtOwHwFnc13tQdRRRSKAvzjMEMzv/zTbrMoTFDsuvBLuPz1ueynBLDUZnKM6c0EY6nLPB3dekLuYopUx09A2H1+NMQRwZmWVeu5mgdCCQWbuj7mjq4R/W/2aD+dNsLP0e3+i8mbef1oatT4V0SEPcpyFp601Kc+8EJkCz3O0+cOsuAcWEKJjJbKWFrgE+dz92m4fCVG7dMrFUAquUuNxTx3K361Wqka4+ayux5THMqHTMPn/gAbq4m7r2D3ne/W0h5il1JAsH9hnxl2jiHunUHijd27VAaFA6SK7Xqk73tAZZFX8DWyqiuRN8VZge/X/MdCRoDw600G0mW34t9+RAyz3cVgeAr7TfUBNXMUUikPUBlUHXQarUg9QX6j58mxtOuJoTIq8Ta9k9bOrnO/taWaHsIFl1csb2lLuCCtFBXZeeCaZb7k63PyP5wGO3EMKRsU9/9h04wDJlH2rFsfjK53KSZQ9fDd0LaM3D8o0h7tOQ3i+4NA+03IsXZlqR/hpqTK2H+NyF9hCwuihXUzkW9yQ2KXG7vcPvPEm4CjUXSay7z9oKbX6S/akyqhetHHiAp5yEYsMbrsvIOz+U+sYGSkQAW8WiAa+VF3rplk7iAU0EHM3vUK9UsKnoHJZ0/4tITyfraju5xPQvEt4amP9BuPgPEKjne6Y/0N4z9cQ9m+We7NLEvbC8pneb2V1CgQjRERjatRLc9jwAsVln9G08/jMIJCvVdTR0Dx1fOLDlbbwign/xGRnbzQVVlIlOGrr042M9qCgU+TNrL6xmhbged8pWpRo58K6231k3wCceIHDqf1MpArgUl2G5G0wM/dvqlh9quZccYkUWzqaK5kyfu92nZYdYXVQkEzl1y8RlElNK4HNNnTFxHr/mD1f79ZexNG9inZzPCbOzZPUIQchZTRVNvWMMsxGq03LSfTOPHvBamddOm/Qhgy0kkyqzw+/R7DsW14mfwi4S7Hr5Xvbu2sIppq2Yj/+k9kCuPhHOuIEzE69S1LVhwDmtJmve3DJSSqLJgeIuehqJYMPs6st7t+mdOLs7MrOwQrEkoX5VuLEdL9Ak/RTN7teq2ldFwl7EYlHLruZhXDMHXgfA0i+NFcBeVEUpXTR06IIdCxDCSaV/4FSnVHoQSJYqVUtLumeQdn+lx5wDgCehGOJuMDH0We6WvoBquENLuyvOzNrAX4NHBokH9WBipLNvrJ7VTXk8TkOwIWfVj3GpokgFj31EDUknBUUf9yfT/WV6mnHF22h2LMgcHN4PtaCGmaKF2vYh0iFbtYwMc9nAVrZ2i4luxYcp0s7eHZsoEgGUWSs5+oSzqKUS+5aHqKz9BykE4piP9x247DIACkIDg7n5tNwTqQQSOcAtYw0302Eqylgtptsshzszxf2L973Lp//0tvZZS6m461/jVXUp88v6ufCEgPKlLFZqe9sSZCMYS1IdWE+XbQb4MrtGmgtmYBEqgTZt5pCMBghIe0Ywte8N6Nc+JNddTUlKgjvospb3pvUKbwWNttkUxyJGQNVgYugdZadY8aZFNF2ZWjJQ3AGcIW35nCnuLiqSScLJMD2JkZdgD0WCFEKacNmmjrhjdRPHgimiibvaoFnF5qrjBj+keI6W6942uLg7A7uJCZtWMJaFsKUQe6yD5i0vA1C59AxMJoXdFeezILqRD8Weot5/IhT0G4TmnUEKBU9soKssnwHVqKqtYOymTHF3x5oJWjPHKTr9mhssEcgU960NAd7Zr7mjaNyILdHNW8qxVPoyRdcy4xgWKHXsbRw8nXLd/nZOULYTnZHNraalZcY6teZv8VAXPdLJjIKBq0nFrov7IW6ZAx1hjmIfIX9mx9VQ1WnMTQZoCOZutTtWDHGfhqS/4H6ns695U6ueBll8iFvGPxsAT0Tvchjp7CswsrooT2rL5Mbg+P3uyVQSVYCQJtxTSdyFoEfxYYlpYtGy8x0AqhYfOge+D1fFfBwiTnvzgayvSykpjtTSYZ8JSvavWcJeiCvZiVL3NgHclOruh7LVVwBQIgLEj7488yCThZCthBK1uS+VTyefqZCxpHbd/pZ7uoAp5ijP2FfRM1fUnj7XRSSu0tITQ5Di1X/+DV74HgBNRStRlEMGb5Qvw0qSSNO2Qe9n79a1FIog/kVnDHzRq/eYD+jiHu4eUMCUxpQW90MCqrvrGpktmjDpPfrTlBx7LmVqgu54J2pq+PYUE4kh7tOQ9Bfc7+jnQ2zbCWY7FBxiRfq1LIaSRIPWPEy33Lc1BmiImKhQtQ9oLvzu6YeOkGZs5qn10QubC7DrzcMiB95lf6qMFQtnDbq/omfMxFqyp0N2hOLUUEfYNy/r6wApZwleGaA6sJ6DrqN7HwJLFi9lvbKEHulgxsmXDjgu4pxBlWgbkOtuM9mIp+J5KX1PW+79fe6BSJxSOkh5DhnYoVfninBfAdfBzjAfVN7hTcf1XNf4ddSGDfzO9AlKy7MM4tB93K6OLYO6C9V9mr/dNjfLEBCvdk5zqAkpJWokoFnu/oHibnbqgf9DxL1j73oUIfHPXZGx3bfwdIpUiUTSkeN22aNlan3DDHJCWkS9tn4f1tYdUDQ/s4wewOYhYvFTLZq15mG6uN/wt03cs66NirTlnoOMmURKS7cUWKbEdPj+RK1+3KrW3tjduZX91nmUeoYI+hZqKx7Rmb2Q6UBTC1WiDXGoG6wfJncJCpKZNBKb0bdKEELQetYv+Mey32J3DkwZTXqqmCHaaA9lWulWk1bglA/XTLoy1mbuE/eWpjosQsVccIhA2wtQUXrdYAC17WGuMz+C12njq8kv8505j3Br6Hzm9evS2EvRXJKKndnJfbRmyRqKJlTKutbTYynudTtm4CxGFWaKZTvtoTgiFiA4iOVudeoZNIeIu2zYpL3f6mMPPQDFqhkFBwK5yzIbC4a4T0PSVlSGuGfLlNGJuGcyU7TQGYpCpAtpL2Bnc5CDIUGRmsIsTDmx3NMrCoX8z049FNXuxyMDxHraKU02Ei8ZmOGSga8aFRPO4IGs1mNHrZYp45ox+BQsm6+varVoUaaF+cHVJ/Gpiy/MfmBBNeV00B7I7NiZtprz4ZpJX7O/z71Ld1nZi6ozd1YUwiYftnifZXugPUS1aEU56lxMyy7lgfWayyaruCsmIoVHaRkzWYKqm+q6OU7sJFS2YkDxWPr6cUcp5aKD+s4IpkSQmMmJ1z5wMpjD6UKVYkDzME/XVoKKt3cV0B9/hTYQ/O0d6wdeexIxxH0akrbcfGm3TDwEXQcGZsroJH2zmCVa6O5sByRduIkkVOrDJhSg3FqQE8u9T9zzP17vUFJ61eSWdVoxkW/OiqEPMFsJOcopTzX1NV3rR0z3B/trlg14LY2rUHNXJDBRvXjkI4gtRTWYRYpw+8GM7Wlxz6fl3t/nHm7VxN1XOjCgHLEW4kx09T4Y25rrcYsotpI5XLV6du9+87OJO2CuXMZiZT+7mgamKG7dsZ0q0YZ3/uB/U+mpoJxOGroiWNUQ2LJXTHsdVkI4SPZr+xtNqMyM76bdc1TWh8fiYz4EQN3+1we9/mRgiPs0JBjX2pkWpMW9Ld28apCJ7P4aKkUbkU5NwBti2hc0JLX/V1i9OfW5K8pYx0JPHIq7GK+I0Lj1VQDmLTtl2GMS3lnMEs1Z0yHV5m0kMWEtGdzn7ivWsjZqrQtQbANzrAfDWaqJX7J9f8b2tFtmqljuCb2AyV8+MHaRsPkpoLt3ulSsTXNvCX8Niyu9rJ5XjNWsMLMw+9/FXn0sPhGmtX5gzKNn9xsAOOcOLu7mgirKRQeNHQGsMo7Zkb2ozmO30IODRKSvSdyuxk4WiDrUsqVZj6mYfRpCSiLd29g3RDbVRGOI+zSkO6qLu113y7Qd0lPmECzFczAJidK8GYD9YU18w+jibnbl1HI3iakn7ma9JUNx8xu0iiKKyqqGP6ZoDjNFM7Xtme6R/W0h/F3v0eWYBabBVyklFdo1IuUnjOpenSW6ZauLZ5q0sObFcs8SUCXQSBIFs7d8wP4pZwlF9LUgEOlWDrqP/JaLlnLnFcdjHmTOrtCDqrJpc8Z2NSXxta4jIay9M2uzYSmopEJ00NamuX+sg/Q60tr+2lEjfT73ht3rsYkkrlnLsx5jNlspwIJTNBD61Snsuvk4Dv7iLBLByQ2wGuI+DQnENLHpzZZp2wlCgaK5Wfd3lmnWpbNNCxLt7jFjUgQhqX1Ri4SV9sjI+m8PRVp0zKYsxSJ5xqb3lzmWHXR4BhYdZcNVMY9CEeTJt7dl+N3vf2UjJ4ltOJacN+TxTl8J3R/8JYsuumFU9yp82kPB0pPplsmn5Z4toGoNN9GlFA4M4qOtlIpEgPZgDDUlcYb0VFw9m6u60MkZC0sHHNdL2RJSKPi6MidTbWsMsEzuoLtwGZgHj+0I3wycIkZ3s7ZicHiyzxdIt/1N9QuoRg9odRDF8wd/KJd7Z9DmLcFbOgtVsVHdvY7920Y2dzdXGOI+DemJaV+0QpdL29BZC94qMGe3mG0lWlpfYbc2yHhbl5njZ/p7LXd7KpWTFLtYQltRmKaguLv8mpDYRBJlxjHD7K1h1h+WTfu38cI2rSKxO5yga+OTWISK69hBAqL98J3ymYHZJMNhsdMuCnFGGjI2T7WAqjPWQsCaXaDNnlJ8Ikx7oIfmQJRK2UzEWgRW18guaHUScM5idnJvRhO1tbsbWCL2Y59z8hAHA3p6ZrJJqyL2+PxZd0uP2uufLWNve48oNkzFg7vcSn01tBdWM/NLj5M65xYA4oblbjBeemKaiBY5dcs9UD+gBDsDTwUxLJSFtEKnje0KR8/w4XG5UDFh10V9vKIRS2gZB2bLCL/Ak4i7sM91UL4wS1VjNvR0yBN93fzw6W3Ekynuf+cAZ8q3STjLoDL7sj0XdFrK8B5SpTolUiH1B4yUEn+ylZijLOv+Dr0FQU9HC7XtYapFCwlv9krewYiXLGGxkpkx07bjLSxCxT1vmAC1nuVSndJWPz5/9uExXruZEPaMaUwlwR00OeZlXZH07uMsoSHYwJoDa9gcr2WTzUoyPPxwl1xiiPs0JBSPIlMmCpy6pd5dlzVlqxdFoVkp01rLAi0JOwvK3FT6nUSFHbteyDTeKT9x3fqxWkYePJws0j3dATw1IxTlwrlgdvC5sh3sawtx9xv7eeD1HZxp3oRlyUcGrUzNBT32CoqSmeX7U8Fyd5i1VVkgkqSUTlR3Rdb9nXqztlhXEwc7wswULZiKakZ1TXv1MVSJNmrr9R4xUmJt1KqLqTpx6IP1KtX5QnMHFRYOIu665W7SDZPWQIT5cj/hwiVZ909T460hEA/wlZe+wv/u+hmfrCxnZ8/ukb61nGCI+zQknIiCNONzWCCV0sbrDWW5Ax1WLXMjYfGgYmJ+mZtKn4MINuz6gOzxintITyezWqZOL/deHH5SCKKWAvANH0wFwOqEkz5HWe0TXF4T5JZntrEw+DY2GYNFH57Q2424ZlCaatX+fXXymgp5SEC1ua0Fj4hgGuRzZ/ZoD9N4TysH2rupFO04dPfgSPHoAc1XXlnDprou9rSGWJTcTsBVA8ONcdTdMvPS4u4vzr6b3UwQB+aElvWy7/WH8YgIrrlDr+6uWHwFj13wGA+e/yC3rfo1Qko2x4ee3JVrDHGfhoQTMaS0aOIeaoVUQvO5D0G3XfsSRkxaSti8Ug+VBQ56UjZsaXFXxyfuPbq4223Zc5fzimJCcfixVx+bvfBlMFZ9FWHz8B3HowBc7FyPdPhh1sjz1seC6qnCKlQinX0ZM3lNhUzGMCtmTLqrorOxFtDa62bFqXfi7Gkl2HwAs0ihFM7Ovu8giAotY2YJu7n0d2/y02e3s1zZqbVGHg6zjYS9iFlCW/2YHNmzZWxmEzHFgUUNQ7iDReu+yzZqqDr1iiFPrwiFuQVzWVy0mNU1p7IwnmSbOrnNxAxxn4ZEklGENGtTZQL6l38Yyz3s0qoIu3FT7rXjc1ioLLATlDYscU0sxm25RzW3jN0+dUbsZXDq9bDymtEd4yyEk7+IZ9+z/On9cJbyLmLheWCa4MZoelZJoHFv76Z8umWiajQjmBps0wqYvFkKmABwaeIuIm0k23WL1j94L5+suEuh4liu4WGuLNnNjm0bKBI9eOavHv5YAE8FZqGvfGyDryaTZjcKKeTjX8aR7OaRqm9jsoy8ytpqVlgWSbFbBIgkIyM+bryMSNyFEOcIIXYIIXYLIbLmbQkhLhNCbBVCbBFC3Jfb2zQYDdFkrK8KtFtPMRvK5w7EPdqXsDXpZH6ZZlnPKHAQxo45ron6eEUjEtX8lo5BrKS8c8qXYcHZoz9u5TXgKOT0DV/HHA9MuEsGwFqkCWGkdX/ftjynQvbPcU/o7XQLymqyH6D3l7FGO7D06J01s/WBGY5PPoIomse3AzdzR+WzAIjqwbt59iedpZTEPGgmGfQN7BDbn+T2xEepOXqEAfd+LI6aSQrJ+pbJa0kwrLgLIUzAHcC5wGLg40KIxYfsMx/4L2CVlHIJ8NUJuFeDERJTY5iEblnobU2H9SPrX6z6mI35pZoVU1ngICTtmPXUyvFaHRHdb+l2FAyz52GG3Qurvwo9DWBxwZwzJ/yS2apU8x1Q7d96INmlfe5MvsrsBygKYbMfS6yDokQjKWEe1gDJirsEPv0EouIYFrc/r00RO7St9SAIr3Zv0uYd0hWXFvcOz0J+o17A6nnZ/fNDURN3YpLwduPboz52rIzEcj8R2C2l3CuljAMPABccss//A+6QUnYCSCknbAyJlJKDhxRvGGQSV+OYlbTlXqe1+nUOHWAyF2vBrC7pZoFuuVfqlrtNz08fr2hE41pxlcs5RS338XDC/9PE6agPgWXiRwgW+v20Sw8i0PddyGeFakyNZbhllEAdPYpvyL9FzOrHL7uZKVqIOiuGTC0cEocfrvg7HHU+HPOJkWcp6eJucQ49z7fTOYc2pYjf+K6jtMBDTdEYsr2EmwVxhbebppa4zwD6q2mdvq0/C4AFQojXhRBvCSHOyXYiIcTnhBBrhRBrW1vHNmPwzk13ctkTl7Grc9eYjj8SSKTiWNL9WwL12od4mCCh1+vl7uQHeD51fO9YsyKXlYiw49B71YzX5x5LRrCmJA73NBR3qxM+/xp8+LZJuVyRy0q9LMbS0xdQNStmBCJ/bhndtZFUUxRE6wk4hl4tJh1FFIoA1aKVVEHN+G7A5oHL74Vzbx35Mek+87ZhxN23mAvtf+DBukJWzyseU7vqmNnD8qjKlvYtBOIDm51NBCMR92zv5NAep2ZgPnAG8HHgD0KIAWtvKeWdUsoVUsoVJSUlo71XAC6YdwEOs4Mvvfgl2iJtwx9wBKLKOFZFd8t0149ouet3Wvlu8jP8K3VMb5tVRREIqxun7o4Zr1smnoxhkxKHawqmQuYCZ6Em8pOA3WKiSZTiDPdVqQoh8jZqr39A9UBHmGqaUYcTbGcxRQSoFi3YSkaXKZMTdMt9OHH3Oswc7IjQE02yav7oXTIACbObk8JhUjLFuqZ1YzrHaBmJuNcB/RsyVwENWfb5h5QyIaXcB+xAE/ucU+4q5/Yzb6Mz1slX1nxl3NbkdCQpE33BrUD9iPK2C13awyCdKZPGbHfjU3PjlomrMaxS4pyu4j7JdFrK8MaboF9fm3yN2oslY72fuV0NHVSKNmylg5fnAyieEipEB8UigKUon+I+9Oexf5/3U+YOkz8/CAmLlxXRHuwm+6S5ZkYi7u8A84UQs4UQVuBy4PFD9nkMOBNACFGM5qbZy0Tw1m9Z8n/ncssp32dz22a+89p38jJWbKqSVFOkSGhLZDUJPY0jttyB3kyZNFanF6fMUYVqSrPcXc4pmOd+GNJjr8Qq41otg47NZMtbKmTaLdNyYAcmISmYMfgUKgCbtxSb0HvhjzYNMhek3TL24Sx3TdwXVXgpdo+to2nK5sFDnONKjuGtxrfGdI7RMqy4SymTwJeAfwLbgIeklFuEEDcLIT6i7/ZPoF0IsRV4CfiGlHL8bQSzsDNog3gPZ9kruHb5tTxX+xwbWzdOxKUOSwLRJEIkcJhtEGwCmRo2xx3AYTVR7LayrCrTH253ebHpluF4i5gSqQQWCcogbVwNRkfUpf+7dvWFxKwma94Cqg69IVyoSYuH2cqGXryn+8sAY0uDHC92nzakw5G9I2SatOV+6hhdMgBSHwZyYvFSdnftnhSX8oi+ZVLKp6WUC6SUc6WUP9C33SSlfFz/WUopr5dSLpZSLpVSPjBRN7ydGgA69rzLaVWnAdASnrDknMOOQCQBShKnxd4vx31k5fRPfHk1Xzoz8wvp9ngxA2ZhHrflnpQJzHJqzU49nFHT/65dtb3b8ma59wuoyg590e4f2tVi7TdmkPEGVMeCEPDJh2HVtUPuVuDUxH3VGFIge9FXB8d7NFfV2qaJb/972JlQCxYfS1RaaN/7LoV27YnbFe3K811NHbojCYTQxX2E1alpKnwOHNbMdDSPV4uLW4Vl3KKRlAksWePzBmNCr1JNdezv3ZSvgGpM1XzuSTWFK3iQmOLsrUIdFL0FAVaPFozOBzNP6vO9D8JZR5Xy40uWceo4xF2xa9+jGsXPR+Z+hBLn2BJKRsME10jnngXlfraKmdhbNuPTlzqdsc4839XUoTuSAJHEZbWPuDp1KAoKtD7XZszjzpZJoGKRh509MWXx+Iqok8WU1m/oHTluNVnH7T4bC9Gkli1T2xGmmkbC7lnYhksZdOkC5581un4+k4zdYuKyFdXD7zgEJr22IxHs5gerf5CL2xqWw+6bpiiCDvd8SkK7sAgzHouHzqgh7mm6wnGEksRjtWuZMla35lscI4UFmkVlkqbxW+6omA+/j9yUpchtZUNqLqLh3d5t+bDcpZS9AdVdzT3MFC2IohF0eEx3bsyHv32Ssbg0yz0enDytOjy/aeXL8MkeOpsPUGAvMCz3fnREtBJ/j83R18d9HFaRTU9bNKWU8fvcSWFmjFWIBgModtvYmJqrjdsL6rNA85AKGU9pDxO7yc7upi6qRQuu8qHTILUDCjTjo2gE+x7mWF3aCjgZnjwX8mEp7sXzjgdg/3tv4rf5DZ97PzrDWom/1+YYfgLTSND7aigpiKjjc8to4n7YeQKnLEVuKxtT+lxc3Xq3m+yTLu7ph77dbKetfh9WoWIpGYFgCwGffVbryzPNsXkMcR8Rc5ZoXd+696/Hb/fTFTPEPU1nRBN3t82hDekYh78d6J1pqSS1QpXxkBASizDEPVcUuWy8J2eTQoF6reoxH6mQ6YeJzWQj1qK3BSkc4eCN8qVab5hpjsvtQ5WCVGTyRu0dluJu9/hpNpVjad1Cgc1wy/SnO6qJu02YINgy8qlCg5EWd1WO3y0jZF9DM4NxU+iykjQ5aXPM7hX3fKRCph/6FsWGJaCnZY5U3I8QXHYrQRwQNcR9WLq8C6mI7sZj9RlumX4Eovq4s3gYkOO33PV5pxZ1fEVMCTVFXIBFjHzIgcHQmBTBgnI325R5UP8uSJkXyz3truuJQJVsQlVs4C4f5qgjC7fdTEC6ELHJaRoGh7G4myuXUUMT0aCJqBolnAjn+5amBN1R7YtmjeofovH63BUTCcWOTUptNusYCcWSJITAahpb+bZBdhZXeHktPAsiHdC5P6+We1sgRY1oIuGbNaHDwQ9HXFYzPThRDHEfnrL5K1CEJN6muWQMv7tGj96e15YW9xFWpw6FanbikKlxWe49oTAxRWA1TXyv8yOJJZU+3ojWaL/Ur8tLKmT6c9HcLZklmjGXzJ3U6x8OmBRBULgwJ3om7ZqHrbi7Zx0LgGjThs4afneNYEy33MP632O8ljugWlw45Ph87oFgh3ZfQ4wzMxg9iyu97JBVqCYb1L+L1WQlKZMkU8lJu4f0SqGxM84spQVzkSHu2YgoLqxJQ9yHp2AWUcVFQbfWfdjwu2uEdNeJLdyu5xG7xn1OaXHhlCqx8VjuevGGzeIY9/0Y9HFUuYckZppdC3std5jcaUxpt0ysqxM7cSjMQ/vew4CoyY01GZy06x2+4i4EwYKjOCppWO5pUilJJC3uofbxZ8roSKsLZ0pFHYdFGAxpD1+HZXKGWRwpeOwWaoqcbBPzoXEjVj3VdDLFPR1Q9Ub1RrBGpkxWEmY3dtUQ9xGRKlvC8pTWHMtoQQDBeBIpNPG1hlr7+lWPE2F14ZLaeccarAtFtH8fu238KwmDTBZXenktMguSEW3FxvjbM4+GtOVeEtfb2BrinpW4xYNDhiE1OfMnDmtxT5YupVxGUVAMcQe6wwnQhx/YQh3gLhvmiJGh2Nx4dDvLOPcAACAASURBVHEfq989HNECvE6rMagj1yyp9PFij9bYyqY3i5tMyz39IClLtKAKc06C+NORpMWLgoT45PjdD2txNxXNRQFcisPIlkFv96ukLfd2cOemrajJ7sYttYfGWC3CiJ6943IYI/ZyzeIKLwdkKUlbAbauA8D4RyKOhvS1ZqotdNsqwGRUIWdDpme1RqfOgOwpi1Xv1+DCZog7EIhq7X4BbGocXKU5Oa/Z7sErNUtwrC0I4jHNWvE4hh5pZjB6llR6AUGXtRKbXt6ej4DqLNoIOsafnTVdkelxfpNUpXpYi7tD77TmkhbDLYM2hUno4m6XEty5EXeTw4dfal/gsTYPiyW0QJJzmHmVBqOnxGOj2G2lS7VhTeiZK5NouUfVKBbFQiFhkvY8Dd04DBB6621piPvw2Nxaj2R3ymSIO+lBHQkEQuu9mCNxx1PeF1Ado+WeSGitiG3DTJo3GD1CCBZVeGlLWLDpA1UmVdyTUayKjQIRJGUrmLTrHm4oDu1vkwhNjpfhsBZ3oQuFW1WMVEj6fO5WYdKG2eXILYO7DEd6SPYYA6qJpNYewmozLPeJYEmlj9aYGWtc+/eZVLeMGsOiWPESBoch7oNh0acxxUKTo1WHtbhjMhPGjkeVdMe6ScnJSTGaqgQiSYRIah0hIYeWewU2XdzH6pZJ6oFYq23sU6EMBmdxpZeelB1TTFshTbZbxiqsKEIijoD2vWPFrLuRJ2sa0+Et7kBYOPAkU6hSpWeSUoymKt2RBFZLChsCFLNWoZoLPGWaD5+BbpmucJxUSmZsa+2J8a1HNnHHS7t7t6n6Q8FmMfLcJ4IllV5C2LHFJ1/cY8lY74QtxWmI+2Ck3cjJSXLLHPY5SxHhoiCppel1xbp6h2YfiXRHEljMKlaJNnw4V5353OXYdQHvnwrZ2hNj1a1rmOF3cMXKWVyyoopnNzfxg6e30R1JUOSycs0ZcxFCoKY0sbGajJa/E0FNkYuY4sSZnHy3TFSN9g4+t7iNgOpguJxOItKKOkkDOw57yz1q6hP3Iy2ouq62gw/f/hqv7NTmZ6bF3SZTuXPJAFjspIRmcff3udd3RYirKWIJlZuf3Mrym5/nm3/bxMIyD1etnk17KE5dp2axq1Jvi2C0/J0QTIrA7vL2us8mO6Bq1hdvhrgPjtumtf2VEcNyHxFxk4siPc1uOoq7lJJoIoVJEVjNfc/il3a08IW/riOaSHHNX9fx0OdPJhD9/+2deZwc1XXvv7eququ7unumZ0YarTNICLEjEBICGxPb4IU4D8hiB+INL4TnLY7j4Lx8nh3bH/vlOc/Os51gXp6J8RLb8U4SGWMLnm2CscEgQJgdCRDaYDTSbN3Ta1Xd98etHrVGs7RQV02pfb8f5sNMrUdV3b86de6559Qx0x6277VvMDXAtXqBw8v+jpaUd3j9G9eTMA1ufmAvpy3L8YYNAzz2/AQ33fUsD+4eY6DXwQ/y5LXnHh5+IoNdUSob+YBqMNxl5xZFdt7jjaxtMSEd0hHVdD/+xd3Ksrg6DIiOmchUqrl88DsP8csdB5isufgS0gmTV5++hCvOWc54uc5fff83nLI0x6dfv45rvraVd3z1PnwJqaxLsuZCT5vFPbkI2H9YzH28pN6Y8k6SNYuzrFt5KMZ/ytIctmWwbdcYl5+9HInaVot7iCSzJOWR4bOwqXiVqbBdursvsvMeb2RTFntwcLS4t4ZrZVkyOQlkOyIdcqJS5x1fuY8Hdo1y5XmDLMomcZIWu0dL3Prw82x+SJU4Pn91L1+6eiO5VIKvvP083vBPd1Ooupwg3GB2antKDzSopZaQ9IeYbOp41fDce5wjBTthGpy1opsHd4/i+xIfF1OCZRz3H7nYIuwsFmAJM/IZqt2ect2TGR2WmY2sbVGQDisiSvw47r9pbjJHryxjm33HfU330ckab/3yvTz+/ARfeOO5vO6sw6s6fvyyM7hrxzBPvFDgHReuJpVQGQqnLu3in968gbd95V5Mo0rSb3PMHag7/aR8OdUMBGA08Ny70zM3vV4/mOdrdz+nJlcZLkmVfa8JCZFS8z6ShhV5KqTtQRmbdEJ32pqNdMJkAodEfV8k52tpQFUIcakQ4kkhxA4hxF/Psd3rhRBSCLGxfSbOjUxmyVEmb+cZqYxEddq2UHU9Htw1yr/cvZPrvvcQv/ePv+DJoQI3vnXDEcIOkLQMLj51Ce95xUlTwt7gZWsXcfsHX05XylODam2OuXuZJaSkT7F86O1orFSjK2VhGjOL9jkDPdRcn/t2juALj4QW91CxbFVx0xbRinvVq5L0XAroNNe5EEJE2rBjXs9dCGECNwCvBvYA9wkhNkspH5u2XQ54P/DrMAydDT+pZjzmE12xjblLKRkuVnnuYIlnD0yyfajAA7vGeHjPOLXgdbYvk+TMFd187hVrOP/EFxe3XL0oQ92tKHFvs+dOdimpcUm5aaR/rFSnJzN7DP2cQRWD/+WOAyA8Euh4e5hYQVG2ZMRhmYpbIe0lKRo52vyp6ziqZg47ooYdrYRlNgE7pJTPAAghvg1cATw2bbtPAp8GrmurhfPQKEHQZWViGXMfL9f5i+9s42dP7J9aljQNzlrZzdsuXMW5g3nWrcyzrDuFEMfu2Va9aijibnQtI7VbUmmKF46WauRniLc3WN6dYnHO5q4dB1hn+yTFcZ95G2sSjvouJBCRee5SSqpelbQPJVPXDZqPmpUhUa1BvQIhh7BaEfcVwO6mv/cA5zdvIIRYDwxIKW8RQkQr7mk1aSkrUgxVoolltcqO/QX+9F/uZ89oiQ+8ai3nDORZ1ZdhRU+ahBmO0NX8msqYaHNYxupeRkpKqrVDXsdYqU7vHJ67EIJzBvLc/tgQpw/IqRZwmnBIOspzT0gRmefeeIhkvBpls72D+J2Im+iCKlCdiIW4z+ROTs03F0IYwOeAt817ICGuBa4FGBwcbM3CeTCDV9GsTMbKc7/zqWHe880HSCVMvvWnF7BxVTRZBFXfxZZAm2t8pHqVuBe8Q9kyY+UaaxbPHWdV4v4CnvCxjZkHXjXtIZUJwjIyOs+9cZ6cV6Fq6aJw8+HZXVBENexod+h0Gq24j3uAgaa/VwLNLnIOOBO4QwixE7gA2DzToKqU8kYp5UYp5cbFi9vzlLeCSmsZaVKoFaj79bYc91j59JYnWJyz2fy+CyMTdoCa9LAtp32lBwKcTA7LN6g15U+PTdbpm8f5WD+Yx6ZOzRBa3EMmm7YpyyQJKSPz3BszlrtklVpSi/t8HOrGFH4JglYU4D5grRBitRAiCVwFbG6slFKOSykXSSlXSSlXAfcAl0spt4Zi8TQSTlDT3VP/lPFqNHUb5mN0ss65gz0sz6cjO6fru7hIksn2Zy1kbQspE9SCh2fd83Gq+/nr37wWHv33WfdbtzKPI6rUBNh6AlOoZG2LIikSvozMc29MlsrJGm7yt7euU8s0xD0CnZpX3KWULvA+YAvwOPBdKeWjQohPCCEuD9vA+UhmGuKuIkVxKUFQqNTJpaKNMTe8NTuEJtSZpIXvJ6gFM03HSnU2GU+Q8Kvwq+tn3S9rW5yxyKIqBClL15UJk6xtUZIRi3vgudu+1I06WkA06t1H4Lm3pD5SyluBW6ct++gs277i2M1qnVRQRjNTVymFcUiHlFJSrLoLJu5hNMQwDIFPihoqW2a8XGO9EZT03bsV9j0Iy9fPuO/6pUnuKmlxD5tsymKUNLbvcSDimHtKSortKjHdwZhBAohfngi9auNxn5vmZLrwpSBXi09lyFLNw5fKk4qSStBizQ6p25GPQ01IkJLRUp1zjB0U86dBIgP3fmnW/d60YTFVIUjqipChkrUtJrFJel7kA6opqRt1tIKZ7eMz9T+msvis0M91/It7KkGRNLma8lrjIO7Fquo3mktFO4BYKx0EwE6F8yXzRYaKEFAeZWyiyBliJ+WB34Gzr4RHvg+lmWcIL0351ITAtqIbf/htJJO0mJRpbN+NfEDVlhLh6Loy85F2ctzg/T6FntNDP9fxL+5BvYZcTaXoFeoL342pUFFvEdmIwzLV4gsAJNPhVOaTRg5PCOrjuxFDD2MLFzGwEc77U3Ar8OA3Zt6xXqImBElL1x0JE8MQVAwl7gvhuZsZHZaZj4YmFCpu6Oc67sXdMAQl0qRqk1jCYrI+udAmTd24yGPuk0MApJxwxN23VLywMr6b9P4H1blWXwBLTocTLoStN4E/Qx/b2iRVIbATTih2aQ5RNx1SXnSee7kRCvQliYwu9zsfWVvVhGq83YfJcS/uAGXhkHQnSSfSlJpK0i4UU2GZiGPu1UlV4iDphNQwIaHCPdWJvfSMPMQLspfMomAKxHnXwOhO2H7bkfsFnrttaXEPm7rpkPZrVL0qUsr5dzhGGp570odkVnvu85G1Vah2Uot7a5TNDAm3SCaRiZXnPmNY5umfw/Ub4d5/Bq+9E66qjZh7dklbj9tAJNVDo1x8niWFR3jUOPlQPZzTLoP8IGz+M9j/xGH7udUCrhCh5N9rDsdLZHD8OhKJ64cvIA1xr8kU2ZSexzAfjSQLHZZpkaqRIekWcSyHkhsDz70yx4DqXZ+FkWfg1uvghvPh4e/Dzl/Ckz+BR26G4v4j92lmYh/s+CnM4JVNDaiG5Lmngiyc6vAT9NX2sSN52qGVZgLefDMIAV+77DCBrwX1aOyEFvew8awMKalCY1HE3RsDqlXfwUnq2kHz0RD3KMIyHXE3qlaWVLVEJpGJRVhmojGgOj0sc/BpePZOuPhvYOk6uP2j8IN3Hr6N3QUXfwQ2vhPMYP9qEbZvgW3/Ck//DKQPl3wULvrLw3atBvXswxq4zASFjsq77gZgT2baiP+itfC2H8FXf08J/Os+A16N2q5fKbtCmFylORyZzGBXDzXJzhLuNW88QEoyw7KIw5DHI9mURXc6gR9ByKwj7oZrZUmXJ+PjuQdP5SPE/f6vgGHB+rdAbgmcdAnsvAuQkMyB9OCOT8GP/woe/Dqc8DLYfQ88/xu1rmulEvQD2+Gnn4CuFXD2VVOHrwadqOyQ8smzSRUzr3hlPAxGumZI51q0Fq6+Bb72X+B7Vyu7TBMGV2DrbJnwSWYjbZJd8SokJBRkljW2Of8Ov+X0ZpI89LHXRHKuzhD3RBabGo6VYnRy4fPcCxWXP0zei/kUcOrvqYVuVXnep/yuEnYAw4QTX374zm++GR77D9jy39XDYOV5cNEHYfXvKLE3DHWs0kH4j/dCdgmseSUAtcoEJMJrQt2VUuJeFYLtYhVOZpb63YtPhvfcAwd3QLqXml+BH/+Jbo4dAYadm2qSHVVYxpYwToaMDsvEio64G15CiYxjJGIRlilWXN5vfBe+cz288Tuw9tXw+A+VIG94+9w7CwFn/D6cdrny1s0Z4vaWDVd9E778u/Cdt8C6N8DSs6hWRiGRCM1z77IDz10IHvDWzNmFCacXnE0A1MaeBsJ76GgOYaQypCIU96pXVe0XRXbWdouahaEjBlRl0I0pI6xYhGUK1Tq9TChx/u7VsG8b3P9V6FkFJ76ytYMYxszC3iDVDW/6Hgxsgod/ALf8BdV6MHAZkrh3pw6J+1Z3DXmntRm4DZGxDV1+IGzMVFfknnva9ymbutxv3OgIz71RRtPBjIXnXipX6KKovPQd/w++8YfKa7/kY+2ts969At5ys8qcGdtF9cEvwO6fhOYh51Mq26VkJvi1fxrnzdFir5mpapW6tkzoJJycarNIRJ57rYgtfcq6UUfs6AjPXaTUByslBRWvEkl+71wY5aDGytIzlXftucFA6pvDOaEQ0HMCtfwApjCxjHCe2b2OEvefrX4fe1lMPn10nrsOy4SP7XRPee6RDKjWJ1X7RS3usaMjPHej0fXdU/m9ZbdMLrlwzXrNiso3x1kE/afB238E43tDb6tV9aqhCmhPWon7cFmJ9VzNsafbBdpzj4Kk0xWp516pT2L7krpu1BE7OkLcGzWS7boHwGR9ckHFPVUNPPdM0Epw6VnqJ2SqXpWUGV66YU/aQUrBaFmFvnoyrXnuU3XmteceOqnMIXGPwnOv1kt0S4lra3GPGx0RlkkEfVSTngrHLPSgaqoWpGNmQqrxMgs1rxaqgHalEyAtJqrq+ubT2nOPG9lMBuGrr3UknrtXwZYSqRt1xI6OEHc7KFhkBzXdF3JQ1fMlWa8h7u1pAt4qVa8aqoDalgF+kmJNVQJsNVtGD6hGR9a28KR66EaXCqnFPY50hLinUg41acZC3ItVlz4xgS9MiPgDH7bnLoRAkABRI50wSSVam5HYEPfEXKmdmraQsS3qUoXmIgnLeHVSUmLoRh2xoyPEPWMnKOCQqilPZSErQxarLr1MUE3m25v22AJhe+4ABkkQdXpa9NpBh2WiJJeycH11nSMJy/h1Ej4kbF3OOW50hLg7tklRpkkH4YKFjLkXKnUWiQnqdvSNC2peLXQBNUmCUae7xUwZgJqvwzJRYVtGpJ57RbpImSATcUtJzfx0hLhnkhYFHDJBOGZBPfeKS68o4IXU6m4uwk6FBDCFjXiRnnvC0AIQNkIIaoZDQobvubu+Sw2J8BNkkrpoWNzoCHF3bJMiaTJVJeqN1l8LQaHq0se4ynGPmCjCMglDee49LXruUkruff5e+p3+Q409NKFSNRySEYh7MajTb/hJHF3uN3Z0hrgnTAoyTTbw2BfScy9UXPpEAZFdGHEP23NPGDbCqLecKfPTXT/lgf0P8K6z3xWqXZpD1E2HpJShi3uhFjSj91JHlrfWLDgdIe6WaVBq9FG1FraPammyRJcoYebCnY06E1F47rZpg2hN3Gtejc/e/1lOyp/EH5z0B6HapTmEaznYEYj7RH0CAOmlcXRYJnZ0hLgDVM0MdtBqb9JdOM/dLQRNqruiF/coBlRtM4VoMSzzrSe+xe7Cbj608UOh1bvRHImXyGD7fugDqg3P3fUz2nOPIR0j7hUzi+1NRt5qb3qHeTl5AFgYcY8iLJMy04HnPvd5RiujfPGhL/KyFS/jpSteGqpNmsPxE6qPajXobxoWhaBAXs3N6v6pMaRj7kjdzGC6Ho6Zikzcnx1/litvuZKyW8YQBgYGQgpuGlhB+uHPYT55I67vUvfrCARpK03aSpOyUhjCQCAwhIEvfXzpIzn0oBAIJBIpJRKJL/2pY7m+q84X/AgEQggKtULonruTUJ57Pp1guDTMdf95HSOVESzDImkmMYWJEIKxyhglt8R1G68L1R7NDCSz2FJSCzkluFh8Xv3fz2vPPYa0dEeEEJcC/wCYwJeklH83bf0HgWsAFxgG3iGlfK7Nts5JPZGDKjimHVme+w+f/iFVr8q1665FIPClz7YH7+KE0r2UBk9HJjNYhoVlWEgkZbdM2S1Tdav4+FOCLYTANEwMDBCABIlEIFD/CUxhkjASWIaFaZjqYSAlnvTUQ0HCUmcplwxeEuq/OZ1IBZ67xafu/VseOfAIFw9eTN2vU/NqUw8px3K4+oyrWZNfE6o9miMRySzJqqRaDzdrbKI4pP7v9eHo/qmxY15xF0KYwA3Aq4E9wH1CiM1SyseaNnsQ2CilLAkh3g18GrgyDINnw0+qLu+OkWA0gmwZKSVbdm5h09JN/Nn6P5ta/t279vLHI1vgZX8L6c6rt7E4k0UckDxd+hW3P3c7f37un3PNWdcstFmaJoStmmRPhB2WKQ0jpGTU69P9U2NIKzH3TcAOKeUzUsoa8G3giuYNpJQ/l1I23OV7gJXtNXN+Gn1UM0Y0rfYeH3mcXYVdXLrq0sOW27VR6liqDV4Hckq/mpz1hd98mpN7TubqM65eYIs00zHTqkl2JeT5HoXKCFlfMkKeVKJjhu86hlbuyApgd9Pfe4Jls/FO4MfHYtSLItVotWdEkuf+k50/wRLWEWEQpz5C0exW3ZE6ENtSMf3x6jgff8nH9azTGJJIq1Z7FTfkPPfqODnfp5zs0xPUYkgr71Iz3TU5wzKEEG8GNgIvn2X9tcC1AIODgy2a2CKNJtkYlEOONUopuW3nbZy//Hzy0yo/ZtwxJq08PaFasHA0moG86bQ3cdbi8BuQaI6eRNCNqRLBJKashHE7E+p5NC+OVjz3PcBA098rgX3TNxJCvAr4MHC5lHLGT5WU8kYp5UYp5cbFi9tb67zRjSktYdKdPCJFsZ08fOBh9hb3HhGSAch5Y5QTnVv+9IJlF/DW09962DiDJl4knS66fJ+iF3JYxi2TlqYeTI0prYj7fcBaIcRqIUQSuArY3LyBEGI98EWUsO9vv5nzYwQxbsdXWShhzs7bsnMLCSPBxYMXH7EuL8ep2Z3qt8NiZzEfOu9DOAld4jWupJxuejyfsqyH+j0o+BVs39JpkDFlXnGXUrrA+4AtwOPAd6WUjwohPiGEuDzY7DNAFvieEGKbEGLzLIcLjUSmG1ca2DWVIRBW3N2XPlt2buHC5RfSlTy843vN9elhgtoClPvVaBqkc930+Kqf8GhlNLTzFHwXS9q69EBMaemRK6W8Fbh12rKPNv3+qjbbddSkbZuDdJGuKFEvuSX6aL/IPjT8EEOlIT6w4QNHrCsWC/SKCv4CVITUaBpkMjm6XRWWHKuOsTSzNJTzFPARfopMWnvucaRj8pcytsmwzJOqqDKkYc1S3bbrTgAuWnbhEevKY2pSh9TirllAsqkEKV8J7khlJJRzePUqRUPg+w4ZHZaJJR0j7k7SYlh2k6mMA+F1Yxra8ysc36d79MgJuOWxFwAQ2WgbY2s0zWRsi5SnBHesMhbKOYoTuwCouhkyekA1lnSMuGeSynPPBuIeVsx9f2WEfteDZ+88Yp07ocaSEzkt7pqFI2EaJD01H2G0Gk7MvTiuxL3k5vTs1JjSMeLu2BbDdNNVVh/msMIyQ7UJlngePPufR6zzisMAJLuXhHJujaZVLJlCyPAGVAsTewCYqOXI6f6psaRjxH3Kc/frQIieu19Rnvtzd4N7eL1sGYh7Kq/FXbOw1A2HnBSMVcMJyxSmKkL2sCyfCuUcmmOjY8TdsS2GZZ6Mr7IEwoi5+9LnAB79woL6JOy9/7D1onSQqkyQyXZewTDN8UXNdOjyZGgDqhOTKnlg3O1jeXc6lHNojo2OEXfluXfj+D4QTlhmpHQAVwj6+9cB4oi4u1E+wEFy5NL6NVWzsNQth7wnw/PcS6opzaSf1557TOkYce9KJRgze0gCljBC8dyHDj4JwJKek2DZuiPEPVkdYYRubKtjLqvmOKVq5enzauHF3IOHhvTS2nOPKR2jQoYhSOSXAeCIRCgx9/0jTwHQ330CrH457LkXaoceInZ1hHHRrSvkaRac8fRKFnl1RssHQzl+saaaY+dTWdJ6hmos6RhxB1jU00cFmwxGKGGZ/eM7AejvXavE3avB7num1jv1UQqmjrdrFh43v5pe32OsNh5KEb2JWpGUL1jenW37sTXtoaPEfaDPYZg8DuEMqA4V92JISV/fqXDCS8Cw4JkgJbJWIuuNMml1btEwzfFDeula8p6PJ30mAi+7nRS8MinfZLmOt8eWjhL3wV6HIb8bx/PC8dxLwyzyfKyuZZDMwMrzVNx96DH454tJyhpPO+vafl6N5mjpW6nEHWj/oKrvU/BqmH6CZTreHls6StwHehyGZTcptx5OzL02Rr8UYAQxxtW/A89vg39+JZQO8uHsJ3kqf1Hbz6vRHC0n9Pfie6qJRtsHVcsjFAwBbpLleS3ucaWzxL3XUbnubi2UsMx+t0S/0fQauvY1IH044UJ49y+50z9Tz9bTxIKlXSlKnqqK2nZxL+6nYBhIP63DMjGmo4pCDPQ6bJHdZL1aOJ67rHFeoqmM8MqN8P5tkD8BDINixdWNCzSxwDAEnrkM2NH+sMykEnfPc3RYJsZ0lOfenU5QTPSR8X3KtWJbj12qlygIWJKa1kKvdzUYBpW6R6Hikk1pcdfEA+GsBmCksLe9By4OUzAENT/Lsm7tuceVjhJ3AJHrx/Elk20Oy+wPquD1Z46sG/PCeIUrv3g3ri/ZMKizZTTxIJFfi+37jI4fWZ76WPCLQxQNg4rXxVIt7rGl48Tdzi/DkT5Vv47ru2077v6DTwDQnxs4bPn9z41y2RfuYsf+Ije+ZQOvOl0XDdPEg/Syk+nxffaPt9dzLxX24QuBZfWSMDtOQjqGjrszmd4VoRQPGxrdAUB/z4lTy3614wB/cuM9OEmTf3vvhbzmjHDamWk0L4a+lSfT4/kcmBxu63ELk6opTT5oSq+JJx0n7r1LVoRSPGx/YTcAS/pOA+CZ4SLv+sb9rFrk8B/vvZCTl+Tadi6Nph0MLunD8QzG2zyJaWJSNaVZ5HTNs6VmIek4cV+xKI/0VReator75AtkfJ9M74mMlWq882tbsUyDm64+j7yTbNt5NJp2sTyfxvRSTMhKW49bKKsywv1ZPb4UZzpO3Ad60tR9lZ7VznTI/ZWD9HuSupXh3d94gL2jZW58ywYGep22nUOjaSemIbDIURBeW487EaRWLs9pcY8zHSfuK3rSVH1VzKidMff9tQL9wuLrdz/H3c8c5O/+6Cw2ruqdf0eNZgFJWn1MGoL65IH2HFBKCrUCAAPd+vMfZzpO3G3LxDPUQE87Pfchv8ISy+HmB/ewbmU3f3juyrYdW6MJi3RalcEeGfrNoYXHUiWyMkZRqDGtVb2LjsU0Tch0nLgDGEn1oWtXzN3zPQ7g0W108cjeCa44Z0VbjqvRhE0+ryYy7dn7sFqw9374/Fnw9M9f3AHHdlMwlGys6dPiHmc6UtyTaZVrXmpT/8iR0gE8IfBqWQwBl61b1pbjajRhs2TJqQDsG3oKvDps/nMY3w23fADq5aM7mJTws08yZibBT7C0KxOCxZp20ZHinutSnvVEYagtx9s/qjowDY1leOmaRfR36Vl5muODk5asAmB0fBfc808w9DBc8B4Y3Qm/+N9Hd7BHOLnmuAAACl5JREFUfgDbb+Px9KkI6WAYuuNYnOlIcc/3DgIwPNYecR86qMR9/2QPV5yzvC3H1Gii4NR+NbGuPvks3PEpOOV18Nr/CeuuhLs+D8NPtXag0gj8+L/B8nPZLvpICJ0lFnc6Utz7lg7g+D5jxfZkCDTa65W85Vx6pp6Fqjl+6EurdMWqPwEIeN1nQAh4zf+ApAM/+mBrA6y3fQQqY3D59ZS8SdKmbq8Xd1oqYSiEuBT4B8AEviSl/Ltp623gX4ANwEHgSinlzvaa2jqLlw3iPCAZKu/nyZEn8aSH5MVnCDw2th1TSk49cZ2u1645rrAMi5RvMWoacPFHoDvI8sr2wyUfU+J+wyYwEoAE3wW3quLz0gczCVYSDu6Ai/4Sr/8M6nKSTKJ/Qf9dmvmZV9yFECZwA/BqYA9wnxBis5TysabN3gmMSilPEkJcBfwv4MowDG6FRYuXkfd97vd38/ofvr4txxxwXS4+X7fQ0xx/JM1efm3k2bH6jbC/qRT24BvoPfdZkkHdJABpmEgziTTVLG/p1sGrUu2/iCeWvo3nt+4Go0K3rUsPxJ1WPPdNwA4p5TMAQohvA1cAzeJ+BfDx4PfvA18QQggZRtv1FjBMk78ZrrPXKmJJsIWLcYyWZOpZ1p2ms2Q0xx9Zu4/txRqv+vwvZ1j7kuCnBR5U6ZSZtWWW6tmpsacVcV8B7G76ew9w/mzbSCldIcQ40AccFvQWQlwLXAswODj4Ik1ujTNf8SHWPPtrJu3FFKw+SkYGePGj+/lVZ2NbZvsM1Ggi4qS+JUixi/dduP6o9hNAwhRYhkHCMsjaFjnb5I9vq3JS3+JwjNW0jVbEfSZFnO4Ht7INUsobgRsBNm7cGKpXn3zpu0i+9F3ooqSa33YWOb08PvIol5997JlepXoJT3rkkroKatxpJVtmD9DcoWIlsG+2bYQQFtANtGcGkUajOSbydp7R6ijtiJI26spkkzpbJu604rnfB6wVQqwG9gJXAW+cts1m4GrgbuD1wM8WKt6u0WgOpzfVi+u7XP7vlx+xTgiBCF68Gxll07+6Qhx6Ma97dQByCe25x515xT2Iob8P2IJKhfyylPJRIcQngK1Sys3ATcDXhRA7UB77VWEardFoWueVA6/ksYOP4fruEWI+m5DPJPiNdRuWbOC8pedFZb7mRSIWysHeuHGj3Lp164KcW6PRaI5XhBD3Syk3zrddR85Q1Wg0mt92tLhrNBpNB6LFXaPRaDoQLe4ajUbTgWhx12g0mg5Ei7tGo9F0IFrcNRqNpgPR4q7RaDQdyIJNYhJCDAPPAYuYVj0yRsTVtrjaBfG1La52QXxti6tdEF/borDrBCnlvGU5F0zcpwwQYmsrs60WgrjaFle7IL62xdUuiK9tcbUL4mtbnOzSYRmNRqPpQLS4azQaTQcSB3G/caENmIO42hZXuyC+tsXVLoivbXG1C+JrW2zsWvCYu0aj0WjaTxw8d41Go9G0Gyllyz/ApcCTwA7gr5uWfzNY/gjwZSAxy/6rgV8D24HvAMlg+buAh4FtwF3A6TPsew6q09OjwG+AK5vW3QQ8A1SAIvDRdtnWtP71qL6wG2fZ/+pg3+3A1cEyB7gXqAY/d8TBrmD5Haim5pXAtk/GyLZPNV2zJ4FFEdv1E2AMuGXa8huAWrDvJxbges1oV9N3cxTwaeN3sw3X7JGmz9gE8O9R2cXcmrEaeCK4nwXgw1Fes3lsuwl4KFj+fSA70/nn+zkaYTeBp4ETgWRw8tODda9DNckWwLeAd89yjO8CVwW//9/GdkBX0zaXAz+ZYd+TgbXB78uB54F88He+ybbPB+vaYlvwdw64E7hnlhvVi3q49AI9we89QBbVmvBEIIN68PzXhbYrWHcHqvdt2+/nMV6zJOACG4Lf9wM3RGVXsM0lwGU0CRXq878buAjYifriR3a9ZrOrybY9wL8Fn7FI7+U816xZM8YIHjwRfcbm0ozvAUOBbTcG1y/Kz/9ctjXr4Wdpelgfzc/RhGU2ATuklM9IKWvAt4ErAKSUt8oAlKe6cvrOQvXouhj1JAL4GvD7wf4TTZtmgCMGAqSUT0kptwe/70N96RuJ/Keh3iaeBWzUG0BbbAv4JPBplAcyE68FbpdSjkgpR4HbUZ7UWcAjwTWbRH3pXhsDuwC6gN1h3M9jtG0Tyvs8ANRR93RZhHYhpfwpyptrZhPwmJTyF8Hf/0a012s2uwAuANLAe4K/o76Xc12zHVLKZ1DfSxtIRWXXbJoRHPfVwEOBbV8BSkR4zebSs4YeBudIM4MetsLRiPsKlOfSYE+wbAohRAJ4C+oVbTp9wJiU0p1pfyHEe4UQT6MuyPvnMkQIsQnlCTzdZNtq4AXgVNQFb4ttQoj1wICU8pY5TJrt2kwtF0LkgbUoz2qh7QL1RTtFCLFNCPE3tPF+HqNtS1Eez8PAPqAb5dVEZVer9u4j2us1F+8EtkspG9cp6ns5G83X7A9Q93TRQtg1TTP6gDKwq+m4aRboms2gZwghvsIhPbu+leNM52jEXcywbPoT5f8AdzZ5Ny3vL6W8QUq5BvhvwEdmNUKIZcDXgbdLKf2mY9+Jer15HDi/HbYJIQzgc8BfzmbPXPs3lgshLNTr3W3AeAzsAvhH4GZUmOEi4ELicc0s1JvYetT93A2cG6FdR2vvgtolhFgOvAQVu42VbdOO+yeoGHPkds2gGTMdl5jYpg4k5ds5pGdXtnKs6RyNuO8BBpr+XonyXhpGfgz1WvHBpmVbAs/wS6jX7HwgdEfs38S3Ofz1ZgohRBfwI+AjUsp7ptsmpfRQAxsXtcm2HHAmcIcQYifq9XezEGL69OLZrk1j+Y2oQZdHY2IXqMGaASllAfhX4OyY2OYAKSnl08FrcSMWH5VdszHd3uVEe71mYz3QD7wx2N9BjTvFwbY9wIAQog8VojkQtV2zaMYBlKc+2HTcckxsm6JJz/5o+rqWkC0G51Ee1TOo8EdjAO6MYN01wK+A9DzH+B6HD0C8J/h9bdM2lwFbZ9g3CfwU+MC05QI4pcm2z6LiV22xbdo2dzD74OCzKBHqCX7vDa7ZKPBjVBikbdesDXYtCa7ZWuAHqC9iHGwb4PAB1SHgq1HZ1bT+FRw+ONj8+d+JGlCN7HrNZtcMtjUGVBfctia7PozyTqP+/M+oGcG67wefrdUcGlBdcNtQenZS0+9/D/z9XHbMat9RbaxGkZ9CxYaaU4fcYNm24Oejs+x/ImqAYkdwYexg+T+gvNptwM8bF3navm9GDbBta/o5B/X28UvUF66KGtT5RLtsa/XDDbwj2HcH6hUL1NNcciit73ngmhjYlQHuRwlqFZXF8JE4XLNg+fVN1+xxoC9iu34BDKO8uT3Aa5u+wPXgnk4AX4qDXdO+mz5t/G624Zq9DjVYuS9qu5hFM5qO+yQqFXJ6+vSC2cYhPXsY5UB8k6bsmaP50TNUNRqNpgPRM1Q1Go2mA9HirtFoNB2IFneNRqPpQLS4azQaTQeixV2j0Wg6EC3uGo1G04FocddoNJoORIu7RqPRdCD/H91bPVhdL0PzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f35b4e78cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "date='2014-02-04'\n",
    "\n",
    "plt.plot(df_out['y_13'][date])\n",
    "plt.plot(df_out['tpot_test_no_0'][date])\n",
    "plt.plot(df_out['linear_test_no_0'][date])\n",
    "\n",
    "plt.show()\n",
    "\n",
    "#df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_out = pd.read_hdf('results/run_180207/df_out.h5','table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_out = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_test = pd.DataFrame()\n",
    "test_no = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_test['test_no'+str(test_no)] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_no2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [test_no2]\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for P in sizes:\n",
    "        fig, ax = plt.subplots()\n",
    "        fig.set_size_inches(P)\n",
    "        plt.plot(x,m_linear.predict(X)[n1:n1+sample_n])\n",
    "        plt.plot(x,y[n1:n1+sample_n])\n",
    "\n",
    "        ax.set(xlabel='sample no', ylabel='FO flow m3/h',\n",
    "               title='Sklearn Linear model. Training number:' + str(test_no) + '\\nFeatures: \\n '+ str(features_readable))\n",
    "        ax.grid()\n",
    "\n",
    "        fig.savefig(\"results/run_180205/linear_test_no_\" + str(test_no) + str(P) + \".png\")\n",
    "        \n",
    "        \n",
    "        fig, ax = plt.subplots()\n",
    "        fig.set_size_inches(P)\n",
    "        plt.scatter(m_linear.predict(X)[n1:n1+sample_n],y[n1:n1+sample_n])\n",
    "\n",
    "        ax.set(xlabel='predicted flow m3/h', ylabel='measured flow m3/h',\n",
    "               title='Sklearn Linear model. Training number:' + str(test_no) + '\\nFeatures: \\n '+ str(features_readable))\n",
    "        ax.grid()\n",
    "\n",
    "        fig.savefig(\"results/run_180205/scatter_sample_linear_test_no_\" + str(test_no) + str(P) + \".png\")\n",
    "        \n",
    "        fig, ax = plt.subplots()\n",
    "        fig.set_size_inches(P)\n",
    "        plt.scatter(m_linear.predict(X),y)\n",
    "\n",
    "        ax.set(xlabel='predicted flow m3/h', ylabel='measured flow m3/h',\n",
    "               title='Sklearn Linear model. Training number:' + str(test_no) + '\\nFeatures: \\n '+ str(features_readable))\n",
    "        ax.grid()\n",
    "\n",
    "        fig.savefig(\"results/run_180205/scatter_all_linear_test_no_\" + str(test_no) + str(P) + \".png\")\n",
    "        \n",
    "    \n",
    "    # tpot model\n",
    "    \n",
    "    print('Test number, TPOT model: ', test_no, '\\n')\n",
    "    \n",
    "\n",
    "    m_tpot = TPOTRegressor(generations=gen, population_size=50, verbosity=2, n_jobs=cores)\n",
    "    m_tpot.fit(X_train, y_train.reshape(-1,))\n",
    "\n",
    "    score = m_tpot.score(X_test,y_test)\n",
    "    score_perc = m_tpot.score(X_test,y_test)/max(y_test)\n",
    "    print('Score: ',score )\n",
    "    print('Score in % of max: ',score_perc )\n",
    "    \n",
    "    results.append([test_no,'tpot',score,score_perc])\n",
    "    \n",
    "    features_readable = list()\n",
    "    for t in range(len(test_features[test_no])):\n",
    "        features_readable.append(d[test_features[test_no][t]])\n",
    "\n",
    "    x = np.linspace(n1+1,n1+sample_n,sample_n)\n",
    "    \n",
    "    sizes = [[10,6],[12,8],[14,10]]\n",
    "    for P in sizes:\n",
    "        fig, ax = plt.subplots()\n",
    "        fig.set_size_inches(P)\n",
    "        plt.plot(x,m_linear.predict(X)[n1:n1+sample_n])\n",
    "        plt.plot(x,y[n1:n1+sample_n])\n",
    "\n",
    "        ax.set(xlabel='sample no', ylabel='FO flow m3/h',\n",
    "               title='Sklearn TPOT model. Training number:' + str(test_no) + '\\nFeatures: \\n '+ str(features_readable))\n",
    "        ax.grid()\n",
    "\n",
    "        fig.savefig(\"results/run_180205/tpot_test_no_\" + str(test_no) + str(P) + \".png\")\n",
    "        \n",
    "        fig, ax = plt.subplots()\n",
    "        fig.set_size_inches(P)\n",
    "        plt.scatter(m_tpot.predict(X)[n1:n1+sample_n],y[n1:n1+sample_n])\n",
    "\n",
    "        ax.set(xlabel='predicted flow m3/h', ylabel='measured flow m3/h',\n",
    "               title='Sklearn TPOT model. Training number:' + str(test_no) + '\\nFeatures: \\n '+ str(features_readable))\n",
    "        ax.grid()\n",
    "\n",
    "        fig.savefig(\"results/run_180205/scatter_sample_tpot_test_no_\" + str(test_no) + str(P) + \".png\")\n",
    "        \n",
    "        fig, ax = plt.subplots()\n",
    "        fig.set_size_inches(P)\n",
    "        plt.scatter(m_tpot.predict(X),y)\n",
    "\n",
    "        ax.set(xlabel='predicted flow m3/h', ylabel='measured flow m3/h',\n",
    "               title='Sklearn TPOT model. Training number:' + str(test_no) + '\\nFeatures: \\n '+ str(features_readable))\n",
    "        ax.grid()\n",
    "\n",
    "        fig.savefig(\"results/run_180205/scatter_all_tpot_test_no_\" + str(test_no) + str(P) + \".png\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a = linear_model.LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "a."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}