w6_d5 - Model Evaluation & Bayesian Gridsearching.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Evaluation\n",
    "\n",
    "## Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "from sklearn.datasets import load_diabetes\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor as RFR\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.038076</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.061696</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>-0.044223</td>\n",
       "      <td>-0.034821</td>\n",
       "      <td>-0.043401</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>-0.017646</td>\n",
       "      <td>151.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>-0.001882</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.051474</td>\n",
       "      <td>-0.026328</td>\n",
       "      <td>-0.008449</td>\n",
       "      <td>-0.019163</td>\n",
       "      <td>0.074412</td>\n",
       "      <td>-0.039493</td>\n",
       "      <td>-0.068330</td>\n",
       "      <td>-0.092204</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.085299</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.044451</td>\n",
       "      <td>-0.005671</td>\n",
       "      <td>-0.045599</td>\n",
       "      <td>-0.034194</td>\n",
       "      <td>-0.032356</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.002864</td>\n",
       "      <td>-0.025930</td>\n",
       "      <td>141.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>-0.089063</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.011595</td>\n",
       "      <td>-0.036656</td>\n",
       "      <td>0.012191</td>\n",
       "      <td>0.024991</td>\n",
       "      <td>-0.036038</td>\n",
       "      <td>0.034309</td>\n",
       "      <td>0.022692</td>\n",
       "      <td>-0.009362</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.005383</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.036385</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.015596</td>\n",
       "      <td>0.008142</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>-0.031991</td>\n",
       "      <td>-0.046641</td>\n",
       "      <td>135.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0  0.038076  0.050680  0.061696  0.021872 -0.044223 -0.034821 -0.043401   \n",
       "1 -0.001882 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163  0.074412   \n",
       "2  0.085299  0.050680  0.044451 -0.005671 -0.045599 -0.034194 -0.032356   \n",
       "3 -0.089063 -0.044642 -0.011595 -0.036656  0.012191  0.024991 -0.036038   \n",
       "4  0.005383 -0.044642 -0.036385  0.021872  0.003935  0.015596  0.008142   \n",
       "\n",
       "          7         8         9  target  \n",
       "0 -0.002592  0.019908 -0.017646   151.0  \n",
       "1 -0.039493 -0.068330 -0.092204    75.0  \n",
       "2 -0.002592  0.002864 -0.025930   141.0  \n",
       "3  0.034309  0.022692 -0.009362   206.0  \n",
       "4 -0.002592 -0.031991 -0.046641   135.0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=load_diabetes()\n",
    "df=pd.DataFrame(data.data)\n",
    "df['target']=data.target\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df.drop('target', axis=1)\n",
    "y=df.target\n",
    "\n",
    "X_train, X_test, y_train, y_test=train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf=RFR().fit(X_train, y_train)\n",
    "\n",
    "y_pred=rf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3424.3469369369363"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MSE\n",
    "from sklearn.metrics import mean_squared_error as mse\n",
    "\n",
    "mse(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "58.51791979331576"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# RMSE\n",
    "\n",
    "rmse=mse(y_test, y_pred)**0.5\n",
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.17915384581349303"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# RMSLE\n",
    "from sklearn.metrics import mean_squared_log_error as msle\n",
    "\n",
    "rmsle=msle(y_test, y_pred)\n",
    "rmsle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45.38828828828827"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MAE\n",
    "from sklearn.metrics import mean_absolute_error as mae\n",
    "\n",
    "mae(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45765786229520544"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# R2\n",
    "from sklearn.metrics import r2_score as r2\n",
    "\n",
    "r2(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.45765786229520544"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_wine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>14.23</td>\n",
       "      <td>1.71</td>\n",
       "      <td>2.43</td>\n",
       "      <td>15.6</td>\n",
       "      <td>127.0</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.06</td>\n",
       "      <td>0.28</td>\n",
       "      <td>2.29</td>\n",
       "      <td>5.64</td>\n",
       "      <td>1.04</td>\n",
       "      <td>3.92</td>\n",
       "      <td>1065.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>13.20</td>\n",
       "      <td>1.78</td>\n",
       "      <td>2.14</td>\n",
       "      <td>11.2</td>\n",
       "      <td>100.0</td>\n",
       "      <td>2.65</td>\n",
       "      <td>2.76</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.28</td>\n",
       "      <td>4.38</td>\n",
       "      <td>1.05</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1050.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>13.16</td>\n",
       "      <td>2.36</td>\n",
       "      <td>2.67</td>\n",
       "      <td>18.6</td>\n",
       "      <td>101.0</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.24</td>\n",
       "      <td>0.30</td>\n",
       "      <td>2.81</td>\n",
       "      <td>5.68</td>\n",
       "      <td>1.03</td>\n",
       "      <td>3.17</td>\n",
       "      <td>1185.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>14.37</td>\n",
       "      <td>1.95</td>\n",
       "      <td>2.50</td>\n",
       "      <td>16.8</td>\n",
       "      <td>113.0</td>\n",
       "      <td>3.85</td>\n",
       "      <td>3.49</td>\n",
       "      <td>0.24</td>\n",
       "      <td>2.18</td>\n",
       "      <td>7.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>3.45</td>\n",
       "      <td>1480.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>13.24</td>\n",
       "      <td>2.59</td>\n",
       "      <td>2.87</td>\n",
       "      <td>21.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>2.80</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0.39</td>\n",
       "      <td>1.82</td>\n",
       "      <td>4.32</td>\n",
       "      <td>1.04</td>\n",
       "      <td>2.93</td>\n",
       "      <td>735.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0     1     2     3      4     5     6     7     8     9    10    11  \\\n",
       "0  14.23  1.71  2.43  15.6  127.0  2.80  3.06  0.28  2.29  5.64  1.04  3.92   \n",
       "1  13.20  1.78  2.14  11.2  100.0  2.65  2.76  0.26  1.28  4.38  1.05  3.40   \n",
       "2  13.16  2.36  2.67  18.6  101.0  2.80  3.24  0.30  2.81  5.68  1.03  3.17   \n",
       "3  14.37  1.95  2.50  16.8  113.0  3.85  3.49  0.24  2.18  7.80  0.86  3.45   \n",
       "4  13.24  2.59  2.87  21.0  118.0  2.80  2.69  0.39  1.82  4.32  1.04  2.93   \n",
       "\n",
       "       12  target  \n",
       "0  1065.0       0  \n",
       "1  1050.0       0  \n",
       "2  1185.0       0  \n",
       "3  1480.0       0  \n",
       "4   735.0       0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=load_wine()\n",
    "df=pd.DataFrame(data.data)\n",
    "df['target']=data.target\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df.drop('target', axis=1)\n",
    "y=df.target\n",
    "\n",
    "X_train, X_test, y_train, y_test=train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svc=SVC().fit(X_train, y_train)\n",
    "\n",
    "y_pred=svc.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy\n",
    "from sklearn.metrics import accuracy_score as acc\n",
    "\n",
    "acc(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svc.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4573643410852713"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Precision\n",
    "from sklearn.metrics import precision_score as prec\n",
    "\n",
    "prec(y_test, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3888888888888889"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recall\n",
    "from sklearn.metrics import recall_score as rec\n",
    "\n",
    "rec(y_test, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.27602905569007263"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# F1-score\n",
    "from sklearn.metrics import f1_score as f1\n",
    "\n",
    "f1(y_test, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2, 10,  0],\n",
       "       [ 0, 16,  0],\n",
       "       [ 0, 17,  0]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Matriz de confusion\n",
    "\n",
    "from sklearn.metrics import confusion_matrix as cm\n",
    "\n",
    "cm(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZgU5bn+8e89AyqLiiI7CChuGI1G5bhHXMGjYnLigntiglE56jGJmvyMJrjEJeEkHo0JOS64sWg0QcUtLnE5LqAhKiACojIzCIhbVJaZnuf3RxfYMwwzPdAzPTVzf7zqurqq3qp6ui2eefupt6oVEZiZWXqVFDsAMzPbME7kZmYp50RuZpZyTuRmZinnRG5mlnJO5GZmKedEbhtMUgdJD0r6VNK9G7CfkyU9XsjYikHSI5JOL3Yc1nY4kbchkk6SNF3S55IWJQln/wLs+jtAD6BrRBy3vjuJiLsj4vACxFODpIMkhaQHai3/erL8mTz38wtJdzXULiKGR8T49QzXrNGcyNsISRcCvwWuJpt0twZ+D4wowO77A29HRFUB9tVUlgL7SOqas+x04O1CHUBZ/jdlzc4nXRsgaXNgDHBuRNwfEV9ERGVEPBgRP0nabCzpt5Iqkum3kjZO1h0kqUzSjyQtSXrz303W/RK4DDgh6emfWbvnKmlA0vNtl8yfIekdSf+StEDSyTnLn8/Zbl9J05KSzTRJ++ase0bSFZJeSPbzuKSt6vkYVgF/AU5Mti8FTgDurvVZ/U7SQkmfSXpV0gHJ8mHAz3Le5z9z4rhK0gvAl8A2ybLvJ+tvlvTnnP1fK+lJScr7f6BZA5zI24Z9gE2AB+pp8/+AvYHdgK8DQ4BLc9b3BDYH+gBnAjdJ2iIiLifby58UEZ0j4pb6ApHUCbgBGB4RmwL7AjPqaLcl8HDStiswFni4Vo/6JOC7QHdgI+DH9R0buAM4LXl9BPAmUFGrzTSyn8GWwD3AvZI2iYhHa73Pr+dscyowCtgUeK/W/n4E7JL8kTqA7Gd3evjZGFZATuRtQ1fgwwZKHycDYyJiSUQsBX5JNkGtVpmsr4yIqcDnwA7rGU818DVJHSJiUUTMrKPNvwNzI+LOiKiKiAnAW8DROW1ui4i3I2I5MJlsAl6niPg/YEtJO5BN6HfU0eauiFiWHPM3wMY0/D5vj4iZyTaVtfb3JdnPcSxwF/CfEVHWwP7MGsWJvG1YBmy1urSxDr2p2Zt8L1m2Zh+1/hB8CXRubCAR8QXZksYPgUWSHpa0Yx7xrI6pT878B+sRz53AaGAodXxDkfRjSbOTcs4nZL+F1FeyAVhY38qIeBl4BxDZPzhmBeVE3ja8CKwEjq2nTQXZi5arbc3aZYd8fQF0zJnvmbsyIh6LiMOAXmR72X/KI57VMZWvZ0yr3QmcA0xNestrJKWPi4DjgS0iogvwKdkEDLCucki9ZRJJ55Lt2Vck+zcrKCfyNiAiPiV7QfImScdK6iipvaThkq5Lmk0ALpXULbloeBnZUsD6mAEcKGnr5ELrT1evkNRD0oikVr6SbImmuo59TAW2T4ZMtpN0AjAYeGg9YwIgIhYA3yR7TaC2TYEqsiNc2km6DNgsZ/1iYEBjRqZI2h64EjiFbInlIkn1loDMGsuJvI1I6r0Xkr2AuZRsOWA02ZEckE0204HXgTeA15Jl63OsJ4BJyb5epWbyLUniqAA+IptUz65jH8uAo8heLFxGtid7VER8uD4x1dr38xFR17eNx4BHyQ5JfA9YQc2yyeqbnZZJeq2h4ySlrLuAayPinxExl+zIlztXjwgyKwT54rmZWbq5R25mlnJO5GZmKedEbmaWck7kZmYpV98NIkW1a899fBW2iT26zSbFDqHV23p6wZ7JZfWoWlW+wc+uqfzwnbxzTvuttmlRz8pxj9zMLOVabI/czKxZVWeKHcF6cyI3MwPItOTH6dfPidzMDIio60kR6eBEbmYGUO1EbmaWbu6Rm5mlnC92mpmlnHvkZmbpFh61YmaWcr7YaWaWci6tmJmlnC92mpmlnHvkZmYp54udZmYp54udZmbpFuEauZlZurlGbmaWci6tmJmlnHvkZmYpl6ksdgTrzYnczAxcWjEzSz2XVszMUs49cjOzlHMiNzNLt/DFTjOzlHON3Mws5VxaMTNLOffIzcxSLsU98pJiB2Bm1iJEdf5TAyQNkzRH0jxJl9Sx/oeS3pA0Q9LzkgYnyw+T9Gqy7lVJB+cTunvkZmYAVYX5YQlJpcBNwGFAGTBN0pSImJXT7J6I+EPS/hhgLDAM+BA4OiIqJH0NeAzo09Ax3SNfh/2G7s2U5yfy0Iv38r3Rp661vv1G7bnuj1fw0Iv3cvfU/6V3v5411vfs04OX5j/J6WefVGN5SUkJk54Yz//c+esmjT+NNv63veg+YTzdJ99F51NHrrW+04nH0e3u2+h2x//S9YbfUNqzBwDtttuWrcbdSLe7sus2OWRoc4feahxx+EHMfPNZ3pr1PBf95Nxih9O8CtcjHwLMi4h3ImIVMBEYUeNQEZ/lzHYCIln+j4ioSJbPBDpI2rihAzqR16GkpISf/epHnH3ShRx74EiGf+swttl+QI023z7paD775F8ctc9x3PnHiVxwac2T/ie/PI/nn3pprX2f/IPjWTD33SaMPqVKStj8x+ez7EeXsOSkM+hw6CG0G9C/RpPKt+fy4fd+yNLTvs/yp//OZuecBUCsWMnHY37F0lO+y7ILL2bz889FnTsV412kWklJCTf87iqOOvoUdvn6UE444Vh22mm7YofVfKqr85/q1wdYmDNfRh29aknnSpoPXAecV8d+/gN4LSJWNnRAJ/I6fG33wby/oIzy9yuoqqzi0b/8jaFHHFijzUFHHMCUyVMBeOKhp/m3/fdcs27osAMpf38R8+e8U2ObHr26ceCh+3H/3VOa/k2kTPvBO1JVVkGmYhFUVbH8b0+xyQH71Wiz6rUZxMrsOb1q5ixKu3cDILOwjExZOQDVHy6j+uNPKOnSpXnfQCswZK/dmT//XRYseJ/KykomT/4rxxx9RLHDaj6N6JFLGiVpes40qtGHi7gpIrYFLgYuzV0naWfgWuCsfPbVZIlc0o6SLpZ0QzJdLGmnpjpeIfXo1Y3FFUvWzC9etITuvbrV0WYxAJlMhs//9TldttycDh078L3Rp3Dzr29Za78XXXEBY6+4keoUD3NqKqXdtiKz+KvPPLN0KaXdtlpn+05HHcmKl15ea3n7nXaE9u3IlFfUsZXVp3efniws++pzKytfRO/ePevZopVpRI88IsZFxJ4507icPZUD/XLm+ybL1mUicOzqGUl9gQeA0yJifj6hN0kil3RxEpyAV5JJwIS6ruC2Juf85PvcOW4Sy79cXmP5gYftx0cffszs1+cUKbLWo8MRh9J+xx34/O5JNZaXdN2SLS77KZ9cdS1EFCk6S63C1cinAdtJGihpI+BEoMbXcEm5Nat/B+Ymy7sADwOXRMQL+YbeVKNWzgR2jogaDy+QNJZsAf+aujZKvp6MAuiz6UC27NijicKr3+JFS+nRu/ua+R69urNk0dI62vRg8aKllJaW0nnTznzy0afssvtgDj1qKP/183PZdLPORHWwcuUquvfsxkGHH8D+h+zLxhtvRKfOnbj6xsv52ehfNvfba5EySz+ktMdXn3lpt25kln64VruN9vwGnU8/hWXnXgCVX51e6tiRLX/9Kz4bdwuVM2c3S8ytTUX5B/Tr23vNfN8+vaio+KCIETWzAo1aiYgqSaPJjjgpBW6NiJmSxgDTI2IKMFrSoUAl8DFwerL5aGAQcJmky5Jlh0fEEurRVIm8GugNvFdrea9kXZ2SryfjAHbtuU/RulQzZ8ym/zb96LN1LxYvWsqwYw/lknMur9Hmmcef55jjj+T1V9/ksKOG8soLrwJwxrFnr2lz9o/P5MsvljPx1vsAuOHqmwHYc9/dOf3sk53Ec1TOfot2fftQ2qsnmaUf0uHQg/n4F1fWaNNu+0F0ufhClv3XxVR//EnOinZsec0VLH/kcVY8/WwzR956TJs+g0GDBjJgQD/Kyz/g+ONHcOppbWjkSgG/xUXEVGBqrWWX5bw+fx3bXQlcWde6+jRVIr8AeFLSXL66ers12b80o5vomAWTyWS4+me/4eYJv6W0tIS/THiI+XMWcM5FP2DWjNk88/jzPHDPg1x94+U89OK9fPrJZ1x01s+LHXa6Zar5dOwNdP3v66C0hC8feoSqBe+y6fe/y6q35rDy+f9j83N/iDp0YMsrf5HdZPFiPrr4UjocchAb7bYrJZttRscjhwHw8VXXUDU3r/KiJTKZDOdfcClTH76H0pISbh8/iVmz3i52WM0nxXd2KpqoliiphOx4ytXDbsqBaRGRyWf7YvbI24pHt9mk2CG0eltPb0OJsIiqVpVrQ/ex/O6f551zOpx8xQYfr5Ca7M7OiKgG1h5IbWbWEqV4NJlv0TczA8jkVSxokZzIzcwg1TVyJ3IzM3AiNzNLPdfIzczSLarTO1DOidzMDFxaMTNLPY9aMTNLOffIzcxSzonczCzlUvzoYydyMzNwj9zMLPU8/NDMLOU8asXMLN3CpRUzs5RzacXMLOX8rBUzs5Rzj9zMLOWqfLHTzCzdXFoxM0s5l1bMzNLNww/NzNLOPXIzs5RzIjczSznfom9mlm7+zU4zs7RzIjczSzmPWjEzSzn3yM3MUs6J3Mws3SLj0krBzfro/WKH0Op1ffKOYofQ+u1+WrEjsHy5R25mlm4efmhmlnZO5GZmKZfeErkTuZkZQFSlN5OXFDsAM7MWoboRUwMkDZM0R9I8SZfUsf5CSbMkvS7pSUn9a63fTFKZpBvzCd2J3MyM7MXOfKf6SCoFbgKGA4OBkZIG12r2D2DPiNgVuA+4rtb6K4Bn843didzMDArZIx8CzIuIdyJiFTARGJHbICKejogvk9mXgL6r10naA+gBPJ5v6E7kZmY0rkcuaZSk6TnTqJxd9QEW5syXJcvW5UzgEQBJJcBvgB83JnZf7DQzg0aNWomIccC4DT2kpFOAPYFvJovOAaZGRJmkvPfjRG5mBkRVwXZVDvTLme+bLKtB0qHA/wO+GRErk8X7AAdIOgfoDGwk6fOIWOuCaS4ncjMzIAo3+nAasJ2kgWQT+InASbkNJO0O/BEYFhFL1sQQcXJOmzPIXhCtN4mDE7mZWVaBEnlEVEkaDTwGlAK3RsRMSWOA6RExBbiebI/73qSE8n5EHLO+x3QiNzOjoD1yImIqMLXWsstyXh+axz5uB27P53hO5GZmFDaRNzcncjMzIDL5jxJpaZzIzcxwj9zMLPWi2j1yM7NUc4/czCzlItwjNzNLNffIzcxSrtqjVszM0s0XO83MUs6J3Mws5aL+H/5p0ZzIzcxo5T1ySXsDlwP9k/YCIiK2b+LYzMyaTWsffngbcBHwKpBp2nDMzIoj08pHrXwWEQ82eSRmZkXUKnvkknZNXj4l6VfA/cDqnyMiIl5v4tjMzJpNa62R31Rrfv+c1wEcWPhwzMyKo1WOWomIAwAk9Y+I93LXSerf1IGZmTWnNPfIS/Jo80Cey8zMUitTXZL31NLUVyPfHtgJ2FxS7o+CbgZs0tSBpckRhx/E2LFjKC0p4dbbJnDd9bWrUpaP51+bybW33kt1dfDtQ/flzG8fUWP95MeeZeIjz1JaUkLHTTbmsrNPYtt+vQB4+90yxvxhAl8sX4EkJlx3MRtv1L4YbyPV2vK53CpLK8DOwLeBLsBxOcv/BZzVlEGlSUlJCTf87iqGHTmSsrJFvPTiVB586HFmz55b7NBSJZOp5uo/TWLc5efRo2sXRl50LQftteuaRA1w5AF7cfwR2UszT7/yOtff9mf+cNloqjIZfvq727n6vDPYYWBfPvnX57QrLS3WW0mttn4uV7fGUSsR8QDwgKT9I+L5ZowpVYbstTvz57/LggXvAzB58l855ugj2szJXyhvznuXrXt1o2/PrQAYtv8ePP3KP2sk8s4dO6x5vXzlSpT8u3txxmy279+HHQb2BaDLpp2bL/BWpK2fy61y+GGO0yWdVnthRIxanwNK+m5E3LY+27ZEvfv0ZGFZxZr5svJFDNlr9yJGlE6Ll31Cj65brJnv0XUL3pj77lrtJj7yd+6Y8iSVVVX87y8vAODdiiVI4odj/oePPv2cYfvvwfe+dXhzhd5qtPVzOc2llXyq9n8DnkymF4Du5IwnXw+/XNcKSaMkTZc0vbr6iw04hLVWJw7/JlNvHsMFp36Lcfc9AkAmk+G12fP51QXfZfzVP+Kpl//JS6+/VeRILW2qQ3lPLU2DPfKImJQ7L+lOoN5Si6R13SwkoEc9xxoHjANot1GfVPx9rCj/gH59e6+Z79unFxUVHxQxonTq0bULi5d9vGZ+8bKP6b7l5utsP3z/Pbhq3ITstlttwR6DB7HFZtmSygHf2JnZ7yxk7113bNqgW5m2fi63xNEo+VqfyAdSTzJO9ABOA46uY1q2HsdssaZNn8GgQQMZMKAf7du35/jjR/DgQ48XO6zU2XlQf95btISyxR9SWVnFo8+/ykF77VqjzXsVS9a8fvbVN9m6V3cA9tttMHPfq2D5ylVUZTJMnzWXbfv2bNb4W4O2fi5HI6aWJp+nH37MV7GXAB8BlzSw2UNA54iYUcf+nmlkjC1aJpPh/AsuZerD91BaUsLt4ycxa9bbxQ4rddqVlvKz75/A2WNuJFNdzbGH7MOgrXtz04QHGbxtf4YO2ZUJjzzDy6/PoV1pKZt17sCV/5m9dLNZ546cdszBnHTRtQAcsMfOHLjnLsV8O6nU1s/lllgyyZeingq/JAH9gPJkUXXUt0EBpaW0kmZf/OOOYofQ6nXafa1xAtYEqlaVb3AWfqHnd/LOOft9cF+Lyvr1llaSpD01IjLJ5ORqZq1SdSOmliafGvkMSW1nDJKZtUmB8p5amvpu0W8XEVXA7sA0SfOBL/jqF4K+0Uwxmpk1uaoU18jru9j5CvAN4Jh62piZtQotsaedr/oSuQAiYn4zxWJmVjQtsfadr/oSeTdJF65rZUSMbYJ4zMyKorX2yEuBzpDid2dmlqfW2iNfFBFjmi0SM7MiyqS4z9pgjdzMrC1I8S+91TuO/JBmi8LMrMiqUd5TQyQNkzRH0jxJaz3SRNKBkl6TVCXpO7XWbS3pcUmzJc2SNKCh460zkUfERw1Ga2bWShTqoVmSSoGbgOHAYGCkpMG1mr0PnAHcU8cu7gCuj4idgCHAkjra1JDPD0uYmbV6BbzYOQSYFxHvAEiaCIwAZq1uEBHvJutqHDZJ+O0i4omk3ef5HDC9D+A1MyugainvKfdHcJIp9xfT+gALc+bLkmX52B74RNL9kv4h6fqkh18v98jNzIBMI9rm/ghOgbUDDiD7aJT3gUlkSzC31LeRe+RmZmRHreQ7NaCc7OO/V+vLV48Cb0gZMCMi3kmedfUXso9KqZcTuZkZBR21Mg3YTtJASRsBJwJT8gxjGtBFUrdk/mByauvr4kRuZkbhRq0kPenRwGPAbGByRMyUNEbSMQCS9pJUBhwH/FHSzGTbDPBj4ElJb5C9n+dPDcXuGrmZGYW9ISgipgJTay27LOf1NLIll7q2fQLYta516+JEbmZG633WiplZm5FJ8S36TuRmZrhHbmaWek7kZmYpl+Kf7HQiNzMD98jNzFKvMbfotzRO5GZmpPuHJZzIzcxwacXMLPWcyM3MUq6hZ6i0ZE7kZma4Rm5mlnoetWKpVNJjYLFDMGsxqlNcXHEiNzPDFzvNzFIvvf1xJ3IzM8A9cjOz1KtSevvkTuRmZri0YmaWei6tmJmlnIcfmpmlXHrTuBO5mRng0oqZWeplUtwndyI3M8M9cjOz1Av3yM3M0s09cjOzlPPwQzOzlEtvGnciNzMDoCrFqdyJ3MwMX+w0M0s9X+w0M0s598jNzFLOPXIzs5TLhHvkZmap5nHkZmYpl+YaeUmxAzAzawmqGzE1RNIwSXMkzZN0SR3rN5Y0KVn/sqQByfL2ksZLekPSbEk/zSd2J3IzM7KllXyn+kgqBW4ChgODgZGSBtdqdibwcUQMAv4buDZZfhywcUTsAuwBnLU6ydfHidzMjGxpJd//GjAEmBcR70TEKmAiMKJWmxHA+OT1fcAhkkT2SQGdJLUDOgCrgM8aOqATuZkZ2VEr+U6SRkmanjONytlVH2BhznxZsoy62kREFfAp0JVsUv8CWAS8D/w6Ij5qKHZf7DQzo3GjViJiHDCuCcIYAmSA3sAWwHOS/hYR79S3kXvkZmYU9GJnOdAvZ75vsqzONkkZZXNgGXAS8GhEVEbEEuAFYM+GDuhEbmZGQWvk04DtJA2UtBFwIjClVpspwOnJ6+8AT0VEkC2nHAwgqROwN/BWQwd0acXMjMLdEBQRVZJGA48BpcCtETFT0hhgekRMAW4B7pQ0D/iIbLKH7GiX2yTNBATcFhGvN3RMJ/ICOOLwgxg7dgylJSXcetsErrv+pmKHlErPvzSda377BzLV1fzH0cP4/qnH11g/6YGHmXj/Q5SUlNCx4yb84qLz2HZgfyqrqrj8V79l9tvzqcpkOGbYIfzgtBOK9C7SrS2fy1HAW/QjYiowtdayy3JeryA71LD2dp/XtbwhLq1soJKSEm743VUcdfQp7PL1oZxwwrHstNN2xQ4rdTKZDFf+5iZu/s0VTLn7j0z92zPMX/BejTb/fvhBPHDnzfx5/E1876TjuO5//gTA4089x6rKSh6482Ym33oD9/51KuWLFhfjbaRaWz+XM0TeU0vjRL6Bhuy1O/Pnv8uCBe9TWVnJ5Ml/5Zijjyh2WKnzxuy32bpvb/r16UX79u0Zfsg3eeq5l2q06dyp05rXy1esIDvsFiSxfMUKqqoyrFy5ivbt29O5U8dmjb81aOvncqFuCCqGJiutSNqR7FjJl5OvC6uXD4uIR5vquM2td5+eLCyrWDNfVr6IIXvtXsSI0mnJ0g/p2b3bmvke3bfijZlz1mo34c8PMn7i/VRWVXHrDdcAcNjQ/XnquRcZOuIkVqxYyUXnjWLzzTZttthbi7Z+LheytNLcmqRHLuk84K/AfwJvSsq9q+nqpjimtQ0j/+NoHr33Ni48+3v88fYJALwxaw6lJSU89de7efS+2xk/4X4Wli8qcqSWNmnukTdVaeUHwB4RcSxwEPBzSecn67SujXLvlqqu/qKJQiusivIP6Ne395r5vn16UVHxQREjSqfu3bbigyVL18wvXvIh3bt1XWf74Yd+k6eeexGAqU88w35770n7du3oukUXdtt1MDPfmtvkMbc2bf1cLuDww2bXVIm8ZHU5JSLeJZvMh0saSz2JPCLGRcSeEbFnSUmndTVrUaZNn8GgQQMZMKAf7du35/jjR/DgQ48XO6zU+dqO2/N+WQVlFR9QWVnJI0/+naH7712jzXsLv7qn4tn/e4Wt+2bveu7VoxuvvPpPAL5cvoLXZ77FwP79sMZp6+dyY27Rb2maqka+WNJuETEDskNqJB0F3Ars0kTHLIpMJsP5F1zK1IfvobSkhNvHT2LWrLeLHVbqtGtXys/+62zOuvBSMpkM3zrqcAZt058b/3QHO++4PUMP2Jt7/vwgL037B+3atWOzTTtz9aU/AmDkt4/m0qvHMuLkswiCY488nB0GDSzyO0qftn4ut8SSSb7UFAV+SX2BqohY63uZpP0i4oWG9tFuoz7p/VRTYnnFc8UOodXr0PuAYofQJlStKl/nN/187dNnaN4558Xypzf4eIXUJD3yiCirZ12DSdzMrLmledSK7+w0MyPdpRUncjMz0v2bnU7kZmZAJvL5Nc6WyYnczAzXyM3MUs81cjOzlHON3Mws5apdWjEzSzf3yM3MUs6jVszMUs6lFTOzlHNpxcws5dwjNzNLOffIzcxSLhOZYoew3pzIzczwLfpmZqnnW/TNzFLOPXIzs5TzqBUzs5TzqBUzs5TzLfpmZinnGrmZWcq5Rm5mlnLukZuZpZzHkZuZpZx75GZmKedRK2ZmKeeLnWZmKZfm0kpJsQMwM2sJohH/NUTSMElzJM2TdEkd6zeWNClZ/7KkATnrfposnyPpiHxidyI3MyPbI893qo+kUuAmYDgwGBgpaXCtZmcCH0fEIOC/gWuTbQcDJwI7A8OA3yf7q5cTuZkZ2Rp5vlMDhgDzIuKdiFgFTARG1GozAhifvL4POESSkuUTI2JlRCwA5iX7q1eLrZFXrSpXsWNoLEmjImJcseNozdL2GVetKi92CI2Wts+4UBqTcySNAkblLBqX85n1ARbmrCsD/q3WLta0iYgqSZ8CXZPlL9Xatk9D8bhHXlijGm5iG8ifcdPzZ9yAiBgXEXvmTEX9w+dEbmZWWOVAv5z5vsmyOttIagdsDizLc9u1OJGbmRXWNGA7SQMlbUT24uWUWm2mAKcnr78DPBXZq6hTgBOTUS0Dge2AVxo6YIutkadUm6srFoE/46bnz3gDJDXv0cBjQClwa0TMlDQGmB4RU4BbgDslzQM+IpvsSdpNBmYBVcC5EZFp6JhK8yB4MzNzacXMLPWcyM3MUs6JvAAauh3XNpykWyUtkfRmsWNprST1k/S0pFmSZko6v9gxWX5cI99Aye2zbwOHkR28Pw0YGRGzihpYKyPpQOBz4I6I+Fqx42mNJPUCekXEa5I2BV4FjvW53PK5R77h8rkd1zZQRDxL9uq+NZGIWBQRryWv/wXMJo+7Cq34nMg3XF234/rkt1RLnsa3O/BycSOxfDiRm1kNkjoDfwYuiIjPih2PNcyJfMOt1y21Zi2RpPZkk/jdEXF/seOx/DiRb7h8bsc1a/GSx6jeAsyOiLHFjsfy50S+gSKiClh9O+5sYHJEzCxuVK2PpAnAi8AOksoknVnsmFqh/YBTgYMlzUimI4sdlDXMww/NzFLOPXIzs5RzIjczSzkncjOzlHMiNzNLOSdyM7OUcyK3gpKUSYatvSnpXkkdN2BfB0l6KHl9TH1PlpTURdI5OfO9Jd23vsc2SxMnciu05RGxW/KEwlXAD3NXKqvR511ETImIa+pp0gU4J6d9RUR8p7HHMUsjJ3JrSs8BgyQNSOzdu4wAAAHdSURBVJ7XfgfwJtBP0uGSXpT0WtJz7wxrnu3+lqTXgG+v3pGkMyTdmLzuIekBSf9Mpn2Ba4Btk28D1yfHfDNpv4mk2yS9Iekfkobm7PN+SY9Kmivpuub9eMwKw4ncmoSkdsBw4I1k0XbA7yNiZ+AL4FLg0Ij4BjAduFDSJsCfgKOBPYCe69j9DcDfI+LrwDeAmcAlwPzk28BParU/F4iI2AUYCYxPjgWwG3ACsAtwgqR+mKWME7kVWgdJM8gm5/fJPrsD4L2IeCl5vTcwGHghaXs60B/YEVgQEXMje8vxXes4xsHAzQARkYmITxuIaf/V+4qIt4D3gO2TdU9GxKcRsYLsL5f3b9S7NWsB2hU7AGt1lkfEbrkLss9i4ovcRcATETGyVrsa2zWTlTmvM/jfhKWQe+RWDC8B+0kaBCCpk6TtgbeAAZK2TdqNXMf2TwJnJ9uWStoc+Bew6TraPwecnLTfHtgamFOIN2LWEjiRW7OLiKXAGcAESa+Tfarhjkl5YxTwcHKxc8k6dnE+MFTSG2R/V3JwRCwjW6p5U9L1tdr/HihJ2k8CzoiIlZi1En76oZlZyrlHbmaWck7kZmYp50RuZpZyTuRmZinnRG5mlnJO5GZmKedEbmaWcv8fyjMj56ZYefMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax=sns.heatmap(cm(y_test, y_pred)/sum(sum(cm(y_test, y_pred))), annot=True)\n",
    "b, t=ax.get_ylim()\n",
    "ax.set_ylim(b+.5, t-.5)\n",
    "plt.title('Confusion Matrix')\n",
    "plt.ylabel('Truth')\n",
    "plt.xlabel('Prediction')\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "      <th>28</th>\n",
       "      <th>29</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0      1       2       3        4        5       6        7       8  \\\n",
       "0  17.99  10.38  122.80  1001.0  0.11840  0.27760  0.3001  0.14710  0.2419   \n",
       "1  20.57  17.77  132.90  1326.0  0.08474  0.07864  0.0869  0.07017  0.1812   \n",
       "2  19.69  21.25  130.00  1203.0  0.10960  0.15990  0.1974  0.12790  0.2069   \n",
       "3  11.42  20.38   77.58   386.1  0.14250  0.28390  0.2414  0.10520  0.2597   \n",
       "4  20.29  14.34  135.10  1297.0  0.10030  0.13280  0.1980  0.10430  0.1809   \n",
       "\n",
       "         9  ...     21      22      23      24      25      26      27  \\\n",
       "0  0.07871  ...  17.33  184.60  2019.0  0.1622  0.6656  0.7119  0.2654   \n",
       "1  0.05667  ...  23.41  158.80  1956.0  0.1238  0.1866  0.2416  0.1860   \n",
       "2  0.05999  ...  25.53  152.50  1709.0  0.1444  0.4245  0.4504  0.2430   \n",
       "3  0.09744  ...  26.50   98.87   567.7  0.2098  0.8663  0.6869  0.2575   \n",
       "4  0.05883  ...  16.67  152.20  1575.0  0.1374  0.2050  0.4000  0.1625   \n",
       "\n",
       "       28       29  target  \n",
       "0  0.4601  0.11890       0  \n",
       "1  0.2750  0.08902       0  \n",
       "2  0.3613  0.08758       0  \n",
       "3  0.6638  0.17300       0  \n",
       "4  0.2364  0.07678       0  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ROC-AUC\n",
    "\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "\n",
    "data=load_breast_cancer()\n",
    "df=pd.DataFrame(data.data)\n",
    "df['target']=data.target\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df.drop('target', axis=1)\n",
    "y=df.target\n",
    "\n",
    "X_train, X_test, y_train, y_test=train_test_split(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "svc=SVC(probability=True).fit(X_train, y_train)\n",
    "\n",
    "y_pred=svc.predict(X_test)\n",
    "y_prob=svc.predict_proba(X_test)[::, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve as roc\n",
    "from sklearn.metrics import roc_auc_score as auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUVf7H8feXIqgouIANkBo60kIgIqBSRCzYBUEBA9iwICKoKIj+XN1dXde1sitiXSyoy9p7T4DQO4Qeamiht+T8/rgXdzYmIYFMbmbm83qeeZhbZu7nZsJ855xzc8acc4iISOwqFXQAEREJlgqBiEiMUyEQEYlxKgQiIjFOhUBEJMapEIiIxDgVghhhZi+Z2UNB5xCRkkeFIEqY2Uoz22tmu8xsm5l9YmY1Dm93zt3inHs0yIwAZlbLzJyfc5efe2Qu+/U3s7lmtsfMNpjZi2ZWKcc+9c3sPTPbbGaZZjbHzO4xs9LFd0Ylh5lNMLNDZnZGLusfy7Hu8OtQJmTd9WaW6r8u683sMzM79yhyDPVfsx1mNt7MyuWxX87fhV2hH1bMrJz/+B3+892T4/GdzWyR/zvynZnVLGxW8agQRJdLnXMVgDOAjcDfw33A0DeSQqrkZ70aeMjMuoY85zDgSWA4UBFoB9QEvjKz4/x96gJTgDVAM+dcReAaIB446SgzHdExnG9YmdmJwFVAJtD3KB5/D/AM8DhwGnAW8ALQs5DPcyEwEuiM95rVAR45wsMqOecq+LfQDytjgDj/ec4H7jOz7v5xqgAfAA8BfwBSgXcKk1VCOOd0i4IbsBLoErLcA1gSsjwBeMy/fx6QDgwDNgHrgQEh+14MzAR24L3RjgnZVgtwQBKwGvgR+AS4I0eeOcAVueQ8/PgyIeumAsP9+ycDu4BrczyuApAB3OQvvwl8UsifUU9gln9ey4DuefzsxgBv5nO+nwFDcjz3bOBK/35D4CtgK7A457mE6fW/0X+t7gLm5dj222uf2+uAV2x3AdcUQY63gcdDljsDG/LY93e/Czm2rwO6hSw/Ckz07w8Gfg3ZdiKwF2gY7p91NN7UIohCZnYCcB2Qks9up+O9AVTDe5N73sxO8bftxntjqYRXFG41s8tzPL4T0Ai4EHiNkE+hZtbcf95PCpC1HdAUSPNXnQOUx/u09xvn3C7gU+Bwy6EL8P6Rnj/kOAnA63itjEpAR7wCUFCh5/svoHfIczfG+9T6if/J/Cu8N8RTgV7AC/4+4dTPzzURaGhmrQvx2ES8n/mHee3gdxttz+d2lr9rE7yieNhs4DQzq5zP8VeZWbqZvep/0sf/XTwjl+dqkttxnHO78Yp7E6TQVAiiy0dmth2ve6Ar8Od89j0IjHXOHXTOfYr3ibABgHPue+fcXOdctnNuDt4bTKccjx/jnNvtnNsLTAbqm1mcv+0G4B3n3IF8jr/ZzPYCyXhdEB/566sAm51zh3J5zHp/O0Blf7mgkoDxzrmv/PNa65xbVIjHh57vh0CLkD7pPsAHzrn9wCXASufcq865Q865mcAkvG6rsPDfhM8H3nbObQS+wSvkBVWZvH/mADjn3nbOVcrnttrftQLe799hh+/n1l23GWiDV0Rb+/u8FfI8oY8/fP+kkO2h23Jul0JQIYgulzvnKuF9uhsC/GBmp+ex75Yc//H34P/nM7O2/uBbhpllArfw3zfgw9YcvuOc24fXP9vXzErhfVp+4whZq/jHG4bXVVXWX78ZqJJHX/wZ/naALf5yQdXA+8R4tELPdydea6eXv6o3/30Dqwm0Df20jFcofvc6mFmHkEHS+f66+SHrOpjZAyHLL+WR7QZgoXNulr/8FnC9mR3+mR7ivz/fw8oC2f5tC3n/zAtrF1733mGH7+/MuaNzbpdzLtUvmBvxfme7mdlJ/vOEPv7w/cPPk/M4ObdLIagQRCHnXJZz7gMgCyj0VR943RqTgRrOG4R9CbCch8mx/BreG15nYI9zLrmAOZ8G9gG3+auTgf3AlaH7mlkF4CK8T7sAX+MNjhbUGqBuHtt2AyeELOdWPHOe77+A3mZ2uFvlu5Dj/JDj03IF59ytv3tC535y/x0kbeKvaxKy7ifn3OMhy7fkkf9GoI5/Zc0G4Gm8QtvD374arz8+VG1gjXMum//+zHN2//3GzPrkuLon5+1w19B8oHnIQ5sDG51zW/J67tAfif9vKefcNrwWX87nmp/bcfwuuboh26Uwgh6k0K1oboQMeOK9affE+yTYxF83gRyDxfk8fhPQz7+f4C/nHDz93QAfsARvkPjhfHL+7vF43SnrgPL+8n14Vz11x/vkWgtvfGAGUM7fpy7eYOyfgdP9dfXwBpEr5XLcBGA7XqEqhTeG0dDf9hZe8SuLd9XR5iOdL1AO2IY3HvDXkPUnAavwPqWX9W9tgEZhet0T/de5GV4BO3x7C5jk79ME7xN0N6A0cCbeoPcTIc8zzP+ZX45XFMviFd4/FTJPd2AD0BhvLObb0OPk2LctXndkKbzuqXeA70K2PwH8AJyCNwC/nv8O8FfF6wq6Cq8QPwmkBP3/MFJvgQfQrYheSO+NfK//H34nMA/oE7J9AgUvBFf7b2Y7gY+B5470xuhvG+Vvq5NPzt89Hq9wzSfkyiO8Pv15/jltBF4GTsnxXA2A9/C6NjLxBg/vBkrncewr8ArVTrzB6Qv99XXwLkXdhdfl82wBz/cVf1ubXHJ9gneV0xb/zbBFmF73l/Df8HOsT8D7lP8Hf/lSYLr/c1qFV0CPz/GYPniXYe7GezP/BDjnKDLd479mO4BX8Yu3v23+4d9LvC61Ff7x1uMN5p8esm85YLz/PBuBe3IcpwuwyP8d+R6oFfT/w0i9mf8DFTlmZnYjMNg5dzTdUSISEI0RSJHwL1m9DRgXdBYRKRwVAjlm/l+TZuA1398OOI6IFJK6hkREYpxaBCIiMa5ETqCVnypVqrhatWoFHUNEJKJMnz59s3Ouam7bIq4Q1KpVi9TU1KBjiIhEFDNbldc2dQ2JiMQ4FQIRkRinQiAiEuNUCEREYpwKgYhIjAtbIfC/dHqTmc3LY7uZ2bNmlmbel463ClcWERHJWzhbBBPwpqTNy0V4X0wdh/f9oy+GMYuIiOQhbIXAOfcj3nzxeekJvO48KUAlMyvMN06JiMSEvdt28MKEr0nfticszx/kGEE1Qr7+D0j31/2OmQ02s1QzS83IyCiWcCIiJcG8Nz9ka72GdBx5C98t3BiWY0TEYLFzbpxzLt45F1+1aq5/IS0iElV2bMhgateraHrDlTgrhf31aW44p3ZYjhXkFBNr8b5Q/LDq/joRkZj21dx11OucSOvN6SRfnUTLV56h+skVwna8IAvBZGCImU3E++7STOfc+gDziIgEasuqdYz+eT0fz93ATZcMovdV7Um8+PywHzdshcDM/oX33bhVzCwdGI33hdg4517C+zLyHnjfHbsHGBCuLCIiJZnLzmb6489R7/FRVDpvAMOGDeHmThdxXJni6b0PWyFwzvU+wnYH3B6u44uIRIIN85aysc8A4uf8wuLaTRg0si81O8YVa4aIGCwWEYk22dmOn8c+S4XWLYhbmErKnQ9Rb/EsanZsU+xZIu77CEREIt2KzbsZMWkOx8/bzr11m1D5rQm0a9k4sDwqBCIixeTQ/gNMu/thUhauZ+F51/PQ8AE0bT0KKxVs54wKgYhIMVj21S9kJyWRuGYxZRK6cP3QjpxW8figYwEaIxARCav9u/eQ3Oc2zureiT9s3cj0v4wjPvmLElMEQIVARCRspq/axu1j3qH1xH8w69yLKL1gAa2HDQq8KygndQ2JiBSxPVu38/n/vcyw45pyZuWazPjiF9p1SQg6Vp5UCEREitC8Ce/zh3vu4PJtG0n/+0fcNKgjFcqV7LfaktU+ERGJUJlrNzH1gitoOuAassqUZdHE/3DnkMtKfBEAtQhERI7Z57PXUr/rObTanE7ydYNp+Y+nqXHSiUHHKjAVAhGRo7R55VrG/LSej+dvJOnSm+l9TXsSu3cKOlahqWtIRKSQXHY20x55hrKNGnLKv15n+IUNGDluJPUisAiAWgQiIoWyYc5iNvXpT5t5KSyq04yBD/ajZvt6Qcc6JmoRiIgUQHa246fRf+WkNi2pt3gWU4aOof7imdRs3zroaMdMLQIRkSNYlrGLkZPmcPzCndwb15wqb71K2+YNg45VZFQIRETycHDfflLvHMWUxRtYcsENPDTiJpq1DH6SuKKmQiAikou0L36EgQNJTF9KmXbduH5oB049ueTMD1SUoqusiYgco307d5PS62ZqXXQ+lbZvZsbT/6RN8hdRWwRAhUBE5DepK7dy2yPv0uq98czodCllFy2k1dCkoGOFnbqGRCTm7dqynS8efZF7jz+bM6vUZNaXySR0jg86VrFRIRCRmDbnlXc49d47uWJ7Buuf+4gBAztyYgTMD1SU1DUkIjEpc816pnW6lLMH9mL/ceVZ8t4nDLn9spgrAqAWgYjEoM9mpdOo6zm03LKW5F630HLcXygfQZPEFTUVAhGJGRnL1/DwTxv4bOEmki6/heuv6UBitw5BxwqcuoZEJOq57GymPfwU5Zo0pso7rzOie0Puf2kkdVUEALUIRCTKrZu1kC19+tNmwVQW1GvO4Idvoka7ukHHKlHUIhCRqJSV7fjxoaeplNCKOmlzmXLvozRcOJ0a7VoGHa3EUYtARKJO2qadjJg0lxOW7KZiw5ac9tYE2jarH3SsEkuFQESixsF9+0m9/X6mLs1gWZcbeHjEAM5uOQozCzpaiaZCICJRYelnP1Bq4EAS16VR5pzuXH93R6qeXD7oWBFBYwQiEtH27dhF8rWDqH3xBVTcsZWZfxtPm18+UxEohLAWAjPrbmaLzSzNzEbmsv0sM/vOzGaa2Rwz6xHOPCISXaYs38KtY9+j9aQJzDi/J2UXL6TlnQOCjhVxwtY1ZGalgeeBrkA6MM3MJjvnFoTsNgp41zn3opk1Bj4FaoUrk4hEh52btvDVYy9yzwktqXFqTeZ8O5WETroa6GiFc4wgAUhzzi0HMLOJQE8gtBA44GT/fkVgXRjziEgUmD3ubU6772567tjChuc+ov/AjpxwnIY7j0U4u4aqAWtCltP9daHGAH3NLB2vNXBHbk9kZoPNLNXMUjMyMsKRVURKuO2r1pHa4RKa39yHfeVPZOkHn3HbbZeqCBSBoAeLewMTnHPVgR7AG2b2u0zOuXHOuXjnXHzVqlWLPaSIBMc5x8cz17C9dVua//o5yX1v54xl82l4ebego0WNcJbStUCNkOXq/rpQSUB3AOdcspmVB6oAm8KYS0QiREbaKh78eSNfLsog6aoh9Lm2A4mdzwk6VtQJZ4tgGhBnZrXN7DigFzA5xz6rgc4AZtYIKA+o70ckxrnsbKaO+hPlmjbhjHde54EeDbn/heHUUREIi7C1CJxzh8xsCPAFUBoY75ybb2ZjgVTn3GRgGPAPMxuKN3Dc3znnwpVJREq+tdPns61vPxIWTWd+XEsGPTKY6gmaJC6cwjrK4pz7FG8QOHTdwyH3FwDtw5lBRCJDVrbj54eeos2fR1HRSjNlxP/R5rERlCpTOuhoUU/D7SISuCUbd3Lf+3OokLaPio3bcMbb42nbOC7oWDFDhUBEAnNgzz6m3zqCacs3s6prP8bcP4DmzR/UJHHFTIVARAKx5ONvKTt4IInrV1D63IvpM7QjlU/S/EBBCPrvCEQkxuzdvpOUKwdQ97KunLh7J7Oee52Enz5WEQiQCoGIFJvkZVu49dH3aDn5TVK7XkX5JQtpcfsNQceKeeoaEpGw27FxM18++iL3VmhFzdNrM/e7VNp2aB50LPGpEIhIWM166U3OHDGUK3ZuZfML/6bfTR05/jhdElqSqBCISFhsXZHO8r6Dif/1M1acXpvtb07klks7Bx1LcqFCICJFyjnH5BlraH5he87etoHkG++k9YtPctwJGgwuqVQIRKTIbFyyggd/3sTXSzaTdPUd9O3VicTz2gYdS45AVw2JyDHLPpTF1JF/5MRmTan+/huMurgRD7wwnNoqAhFBLQIROSbpU+eQecMAEpbMYF6D1gx69Baqta4TdCwpBLUIROSoHMrK5oeRf6JK+zactWoRUx94giYLplKtddOgo0khqUUgIoW2aMMORrw/hxNXHuTkZolUe+sVEhppquhIpRaBiBTY/t17SL7hDr66+mbSt+2l9/0DaDH9O05VEYhoahGISIEs/vdXlLtlMIkbVlK646X0GdqRP1QoF3QsKQJqEYhIvvZsyyTl8n7EXX4hx+/dzewX3yDhh8kqAlFEhUBE8vRr2mZufuwDWn78L6Z1v4YTliyk+S19g44lRUxdQyLyO5nrM/h67PMMq9iG2mfUZv4P02nbvlnQsSRMVAhE5H/M/PsEqj84jJ67trP5pcn0G9CB8mU1SVw0UyEQEQC2rFjDyusH0jrlS5afWZft/3qfmy8+P+hYUgxUCERinHOOj6avpvWF59AscxPJA4YS/8IfKVteg8GxQoVAJIZtWLiMB37N4NulW0i6big39upEYsc2QceSYqarhkRiUPahLKbc9xgVWpxNrfff5OFLGvPAc8OoqSIQk9QiEIkxa1JmsvPGm2i7dBZzG7Vh4OO3cWbL2kHHkgCpRSASIw5lZfP9fX+kaod2VF+TxtSH/kLTeSmc2bJx0NEkYGoRiMSABet2cN+k2Zy8OpuKzc+hxtvjSaivVoB4VAhEotj+3XuYOWgY01dtY8NFSdz24E20aPoAZhZ0NClBVAhEotSiDz7n+Ftvpt2m1dj5Pbn+7o6covmBJBcaIxCJMru3bCfl0r7Uv6oH5Q7sY/Y/JtL2249UBCRPYS0EZtbdzBabWZqZjcxjn2vNbIGZzTezt8OZRyTa/bQ0g5v/7wNafvYu03r0osLSRTQfeF3QsaSEC1vXkJmVBp4HugLpwDQzm+ycWxCyTxxwP9DeObfNzE4NVx6RaJa5dhNfjf07957SjjrV6rDwl5m0bdsk6FgSIcI5RpAApDnnlgOY2USgJ7AgZJ9BwPPOuW0AzrlNYcwjEpVm/G08Z40azuW7t7N93Mf07adJ4qRwwtk1VA1YE7Kc7q8LVR+ob2a/mFmKmXXP7YnMbLCZpZpZakZGRpjiikSWzWkrmdG2C63uTiKzYmVWfvotAwdepCIghRb0YHEZIA44D+gN/MPMKuXcyTk3zjkX75yLr1q1ajFHFClZnHO8P3UVu9u2p8n0H0keOIyz0uZSr3unoKNJhApn19BaoEbIcnV/Xah0YIpz7iCwwsyW4BWGaWHMJRKx1s9bysiULfyQtoVBve+hb+/zSGzfOuhYEuHC2SKYBsSZWW0zOw7oBUzOsc9HeK0BzKwKXlfR8jBmEolI2YeymDJ0DBVbnU29SW/wyGVNuP/Ze6ipIiBFIGwtAufcITMbAnwBlAbGO+fmm9lYINU5N9nf1s3MFgBZwHDn3JZwZRKJRKt/nc7uG2+i7bI5zGnajoFP3sEZZ9cKOpZEEXPOBZ2hUOLj411qamrQMUTC7mBWNj+NeIL2z4xhX9lyLB75KG0euhMrFfTQnkQiM5vunIvPbZummBApgeatzeS+9+dQca1RqXVHarz9Cgl1awYdS6KUCoFICbJv525mDrqHmau3s6nHIO58sD+tmt4fdCyJcioEIiXEwvc/48TbbiYxYw2lLriCPkM7UvHE44KOJTFAnY0iAdu1eRspF19Pg2supsyhg8wd/w5tv/lARUCKjQqBSIB+WJLB4Mc/osWXk5h6yfVUXLqQZgOuDTqWxBh1DYkEIHPNer565O/cW6U9davVZvGvM2nXRl8ZKcEodCEws1JAb+fcW2HIIxLVXHY2M//6CjVH30fPvTvJfPk/9NEkcRKwPLuGzOxkM7vfzJ4zs27muQPvL3/VdhUppM1LVjAroTOt7h3M1j+cxqrPvydJk8RJCZBfi+ANYBuQDAwEHgAMuNw5N6sYsolEhcOTxLW9qD2Ndm4h+ZYRtHlmLGXKaTBYSob8CkEd51wzADP7J7AeOMs5t69YkolEgXVzlzAieQs/Ld/KoL7DufH680hs1zLoWCL/I79CcPDwHedclpmlqwiIFEzWwUNMG/4oZ7/wJA06J9Ft9H30SehBqVIWdDSR38mvEDQ3sx143UEAx4csO+fcyWFPJxKBVv2cyt4bB9BuxTxmN0tk4J/u4vRmmh5CSq48B4udc6Wdcyc7507yb2VCllUERHI4mJXNt0Mf5YzzEjlt42pSH/0bZ8/6mdOb1Q86mki+8mwRmFl54BagHjAHbxrpQ8UVTCSSzE3PZPj7szllQxkqtrmAWm//k/jaNY78QJESIL+/LH4NiAfmAj2Ap4olkUgE2bdjF8nXDOSXawaydfcBBowaQOvkL6isIiARJL8xgsYhVw29AkwtnkgikWHBxP9w0h23kbg5Het2DV8N7UjFE3RJqESe/FoEoVcNqUtIxLdz0xamXHQdjXtfRqnsLOa9/gHtvnhXRUAiVn4tghb+VULgXSmkq4Yk5n23aBMvv/wJr379b5Iv70fz8X+j2ikVg44lckzyKwSznXP6yxcRYNuqdXz9yN8Zfuq5xNWow9KU2SS2bhB0LJEikV8hiKwvMxYJA5edzYy/vEztsffTc+9udv7zPPrccC7lymh+IIke+RWCU83snrw2OueeDkMekRIjY/Fy0nsPoPXMH1l6VkMyJ4znpvMTg44lUuTyKwSlgQr89y+LRWKCc453U1aSePG5NNq5hZTb7if+6TGaJE6iVn6FYL1zbmyxJREpAdbOWsR9U7bwy4rtDLrhPvr1uYB2CWcHHUskrPIrBGoJSMzIOniIaUNH0/zlp2jcJYmLx4ygVxtNEiexIb9C0LnYUogEaMX3UzjYfwDtVi1kVvNzGfjU3ZzW+KygY4kUm/wmndtanEFEituBQ9l8c9cjVOt8LlUz1pH6+HM0n/EDpzWOCzqaSLHSl9dLTJq9ehv3TZpLpYxyVEzsSp03xhFfu3rQsUQCoUIgMWXv9p3MvulOZq/bSeZltzJ81ADiG48MOpZIoFQIJGbMe+vfVLrrNtptWQfdr+PLoR04+XhdEiqS36RzIlFhx8bNTOl2DU37Xg7AvDc+pN1nE1UERHwqBBLVvl6wkYFPTKbZ9/8h5coBVE5b9FtBEBFPWAuBmXU3s8VmlmZmeXbEmtlVZubMLD6ceSR2bF2Rzvv9RzDw9VR21KzH8qnzaDdpPMdXOinoaCIlTtjGCMysNPA80BVIB6aZ2WTn3IIc+50E3AVMCVcWiR0uO5vpT75A3cce5LJ9e9j9Smd69z2X48qo8SuSl3D+70gA0pxzy51zB4CJQM9c9nsUeBLYF8YsEgM2zl/KnFYdiX/gDjadWp113/5Cv/7dVAREjiCc/0OqAWtCltP9db8xs1ZADefcJ/k9kZkNNrNUM0vNyMgo+qQS0bKzHW//sowDHToRtyCV5CGjqLdkNrU6JQQdTSQiBHb5qJmVAp4G+h9pX+fcOGAcQHx8vL4nQX6zZsYChk/ZSsqqTAb3G0m/vp1JbN0k6FgiESWchWAtUCNkubq/7rCTgKbA92YGcDow2cwuc86lhjGXRIFD+w+QOnQ0Lf7xNM27JHHF2JFcG98D/3dJRAohnIVgGhBnZrXxCkAv4PrDG51zmUCVw8tm9j1wr4qAHMnyb34l66Yk2q1exMxWnRj4zL1UbaBJ4kSOVtjGCJxzh4AhwBfAQuBd59x8MxtrZpeF67gSvfYfyuKbO0ZTo1tHKm9Zz/Q/vUiLad9StUGdoKOJRLSwjhE45z4FPs2x7uE89j0vnFkkss1YtZURk+ZyytYTqHhOd+q9OY7WNc8MOpZIVNB1dVKi7dmWSUrPG0m9dhC79h/i1ocGEP/Tx1RSERApMioEUmLNe/0DttdrRLvJb9Csanm+vLsD5zc8NehYIlFHhUBKnMz1GUztfCVN+11FVqnSLJj4HxI/fouTNEmcSFioEEiJ8uX8DSQ9+TFNfvqM5GsGUnXZQhpfd0nQsUSimr6PQEqEzctW890jzzL8zPNpVLseq1Lnk3i2rgYSKQ4qBBIol51N6uPPEff4KHoe2Mu+8RfSq097ypZWY1WkuKgQSGA2zF3Cxj79aTM3mUW1m3L8669yw7maiVykuOljlxS77GzHmz+lcbBjJ+otmknK3aOJWzyTmioCIoFQi0CK1eppcxk+LZMpqzO5OelB+vXtTLsWjYKOJRLTVAikWBzaf4Bpd46i1SvP0LJrElc/+gBXt9YkcSIlgQqBhN2yL3/GJSWRmL6Ema3PY+Df7qNK/RpHfqCIFAuNEUjY7DuYxTe3P8RZF51Hpe0ZzHjqH7RM/Y4q9WsHHU1EQqhFIGExfeUW7ps0l8rbK3Byhx7Uf3McraqfHnQsEcmFCoEUqd1btjO3/xDmbtrDviuGcPvom2hTf0TQsUQkH+oakiIzd/y77KjXkISP36bZqSfwxd0d6FS/atCxROQIVAjkmGWu3cS083vSLOk6DpY9jsXvfky7/7xJhfJlg44mIgWgQiDH5PN560n68yc0+vUrkq8bzKnLFtLomh5BxxKRQtAYgRyVzWkr+W70swyv0ZnGteuxZvp8EpvqaiCRSKRCIIXisrOZNvYZGjw5mssO7mf/qz247npNEicSyVQIpMDWzVrI5r4DSJg/hYV1z+aE18bTt33roGOJyDHSxzg5ouxsx2s/LCX7/POpu3QOU4aNpcGiGdRUERCJCmoRSL5WTZnN8NQdTF2zg3WDHqZ/3wtoe3bDoGOJSBFSIZBcHdy3n+lDHqDVhGdp3XUQ1z32AFe20iRxItFIhUB+J+3zH7CBg2i3dinTE7qQ9NxIqtStHnQsEQkTjRHIb/YdzOLrWx+kVo8LqJi5hRnPvELrKV9Rpe5ZQUcTkTBSi0AAmLZiCyMmzaXyjpOpeN6l1H9jHK2qnRp0LBEpBioEMW7X5m3M73cbczL2ceCqO7hrTBJt4jRJnEgsUddQDJv9z4nsimtIm0/foemZJ/PFXR04N65K0LFEpJipEMSg7avXM63TpTQf1Jv95Y5nyaTPSPzoNU7UJFC7yosAAAyPSURBVHEiMUmFIIY45/hkznpuevpzGqV8Q/L1t3H6sgU0vPLCoKOJSIDCWgjMrLuZLTazNDMbmcv2e8xsgZnNMbNvzKxmOPPEss1LVvB+77u5/a3pHKwbR/rMhSS+9TzlTjwh6GgiErCwDRabWWngeaArkA5MM7PJzrkFIbvNBOKdc3vM7FbgT8B14coUi1x2Nqmjn6bBn8dwadZBDk24jGt6nUMZTRInIr5wvhskAGnOueXOuQPARKBn6A7Oue+cc3v8xRRAf7VUhNbNXMC8pu1o89hw0s+KI+OnFHr36awiICL/I5zvCNWANSHL6f66vCQBn+W2wcwGm1mqmaVmZGQUYcTolJXtePX7pWSffwG1l81nyvDHaLgglRrtWgYdTURKoBLxdwRm1heIBzrltt05Nw4YBxAfH++KMVrEWZk8k2HTdzI9fScbbx5N/xu60LZpXNCxRKQEC2chWAvUCFmu7q/7H2bWBXgQ6OSc2x/GPFHtwN79TL9tJK3feI62XQdyw+Oj6NlCk8SJyJGFsxBMA+LMrDZeAegFXB+6g5m1BF4GujvnNoUxS1Rb+sl3lB48iMR1y5ieeCFJLzxA5dr59cKJiPxX2MYInHOHgCHAF8BC4F3n3HwzG2tml/m7/RmoALxnZrPMbHK48kSjfQez+OrmB6hzaRcq7NzOrL+/RutfP6dy7RpHfrCIiC+sYwTOuU+BT3OsezjkfpdwHj+apSzbzMgP5lJ59ylU7HwFDV5/kRZnVA06lohEoBIxWCwFt3PTFhbceCvztxwg+5q7GDZmAAn1hgcdS0QimC4ojyCzX36LPfUbEf/lJBrX+AOf33Uu59TTJHEicmxUCCLAtlXrSD23B81v6cue408k7cPPSfxgPCeU0yRxInLsVAhKMOcck2evo/9fv6RB6g8k9x3CmcsW0KBn16CjiUgU0RhBCbVp4TJ+HPM37q3Vjeb16rF+9kISG+grI0Wk6KkQlDAuO5tpo/5Mo6fHcnFWFlmvXc7VvdpTupT+MExEwkOFoARZmzqX7X0HkLB4OvPrt6LiG69yXcLZQccSkSinMYISICvb8cp3S6BLF2quWMiUkX+k0fypVFcREJFioBZBwJb/PINhM3Yyc90uNt8ylv43XkDbxpokTkSKjwpBQA7s2cf0W++j9Zsvcs6Fgxjwx1FcerYmiROR4qdCEIAl//mGsoMHkbhhBantL2Lgi6M4peaZQccSkRilMYJitPdAFl8OGkndnt04cc9OZr3wOvE/f6oiICKBUougmPyalsHID+ZRZW8VKna9ikavv0CL0zQ9hIgET4UgzHZs3MzCG25h0daD2HV3M/yRAbSte2/QsUREfqOuoTCa9fwb7ItrSPzXH9KoVlU+v7MDiXUrBx1LROR/qEUQBltXpLO87yDif/2cFWfUYfvb75J4yQVBxxIRyZVaBEXIOcdHM9fS/5mvqD/9J1L63Um1tPnUVxEQkRJMLYIismHeUn4e8wz31r2IFvXrs2nuItrFVQ86lojIEakQHKPsQ1lMe+AJmvztcXq4LLJfv5qrrj1Hk8SJSMRQITgGa6bMZscNA2i7dCbzGsVzyhsTuLZ1k6BjiYgUisYIjsKhrGzGfbMYu7AbNVYvYeqoP9Fk3hSqqQiISARSi6CQlv04jWEzdzNr/W623/4YA27sTEKDOkHHEhE5aioEBbR/9x5m3Dyc+H+9TIcLBzHoiYfp0UyTxIlI5FMhKIBFH35J+VtvJnHjSqZ1uJiklx6m0llnBB1LRKRIaIwgH3sOHOLLm4ZT/8rulN+3m9nj3qbNjx+rCIhIVFGLIA8/L97EyI/mceqB06jY/Voav/Y8zU/V9BAiEn1UCHLIXJ/B4j6DScs8RNnewxgxdgBt62iSOBGJXuoaCjHz2Vc50KAhrb6fTIO4M/nsznNpW0etABGJbmoRAJuXr2F175toNfVrllWLY/s7H5B4UaegY4mIFIuYbhE455g0PZ3+z3xN3TkppCTdw1lpc4lTERCRGBKzLYINc5fw8+i/cm/cxbRqWJ/N8xfTro6+MlJEYk9YWwRm1t3MFptZmpmNzGV7OTN7x98+xcxqhTMPeJPETbn3UU6Kb0GPj1/lqVYn8t4t51BXRUBEYlTYCoGZlQaeBy4CGgO9zaxxjt2SgG3OuXrAX4Enw5UHYHXyDBY1ak3bpx5meb1mbJ86k6uuO18zhYpITAtniyABSHPOLXfOHQAmAj1z7NMTeM2//z7Q2cI0Z8N7ySsoddFFVE9fxtSHn6Lp3GTObNEoHIcSEYko4SwE1YA1Icvp/rpc93HOHQIygd9dr2lmg80s1cxSMzIyjipMrTMqMvGOxzkwZw4Jj9yDlYrpcXIRkd9ExGCxc24cMA4gPj7eHc1ztKn1B9o8mlSkuUREokE4PxavBWqELFf31+W6j5mVASoCW8KYSUREcghnIZgGxJlZbTM7DugFTM6xz2Sgn3//auBb59xRfeIXEZGjE7auIefcITMbAnwBlAbGO+fmm9lYINU5Nxl4BXjDzNKArXjFQkREilFYxwicc58Cn+ZY93DI/X3ANeHMICIi+dOlMyIiMU6FQEQkxqkQiIjEOBUCEZEYZ5F2taaZZQCrjvLhVYDNRRgnEuicY4POOTYcyznXdM5VzW1DxBWCY2Fmqc65+KBzFCedc2zQOceGcJ2zuoZERGKcCoGISIyLtUIwLugAAdA5xwadc2wIyznH1BiBiIj8Xqy1CEREJAcVAhGRGBeVhcDMupvZYjNLM7ORuWwvZ2bv+NunmFmt4k9ZtApwzveY2QIzm2Nm35hZzSByFqUjnXPIfleZmTOziL/UsCDnbGbX+q/1fDN7u7gzFrUC/G6fZWbfmdlM//e7RxA5i4qZjTezTWY2L4/tZmbP+j+POWbW6pgP6pyLqhvelNfLgDrAccBsoHGOfW4DXvLv9wLeCTp3MZzz+cAJ/v1bY+Gc/f1OAn4EUoD4oHMXw+scB8wETvGXTw06dzGc8zjgVv9+Y2Bl0LmP8Zw7Aq2AeXls7wF8BhjQDphyrMeMxhZBApDmnFvunDsATAR65tinJ/Caf/99oLOZWTFmLGpHPGfn3HfOuT3+YgreN8ZFsoK8zgCPAk8C+4ozXJgU5JwHAc8757YBOOc2FXPGolaQc3bAyf79isC6YsxX5JxzP+J9P0teegKvO08KUMnMzjiWY0ZjIagGrAlZTvfX5bqPc+4QkAlULpZ04VGQcw6VhPeJIpId8Zz9JnMN59wnxRksjAryOtcH6pvZL2aWYmbdiy1deBTknMcAfc0sHe/7T+4onmiBKez/9yOKiC+vl6JjZn2BeKBT0FnCycxKAU8D/QOOUtzK4HUPnYfX6vvRzJo557YHmiq8egMTnHNPmVki3rceNnXOZQcdLFJEY4tgLVAjZLm6vy7XfcysDF5zckuxpAuPgpwzZtYFeBC4zDm3v5iyhcuRzvkkoCnwvZmtxOtLnRzhA8YFeZ3TgcnOuYPOuRXAErzCEKkKcs5JwLsAzrlkoDze5GzRqkD/3wsjGgvBNCDOzGqb2XF4g8GTc+wzGejn378a+Nb5ozAR6ojnbGYtgZfxikCk9xvDEc7ZOZfpnKvinKvlnKuFNy5ymXMuNZi4RaIgv9sf4bUGMLMqeF1Fy4szZBEryDmvBjoDmFkjvEKQUawpi9dk4Eb/6qF2QKZzbv2xPGHUdQ055w6Z2RDgC7wrDsY75+ab2Vgg1Tk3GXgFr/mYhjco0yu4xMeugOf8Z6AC8J4/Lr7aOXdZYKGPUQHPOaoU8Jy/ALqZ2QIgCxjunIvY1m4Bz3kY8A8zG4o3cNw/kj/Ymdm/8Ip5FX/cYzRQFsA59xLeOEgPIA3YAww45mNG8M9LRESKQDR2DYmISCGoEIiIxDgVAhGRGKdCICIS41QIRERinAqBSAGZWZaZzQq51TKz88ws019eaGaj/X1D1y8ys78EnV8kL1H3dwQiYbTXOdcidIU/hflPzrlLzOxEYJaZ/cfffHj98cBMM/vQOfdL8UYWOTK1CESKiHNuNzAdqJdj/V5gFsc4MZhIuKgQiBTc8SHdQh/m3GhmlfHmNJqfY/0pePP9/Fg8MUUKR11DIgX3u64hXwczmwlkA0/4UyCc56+fjVcEnnHObSjGrCIFpkIgcux+cs5dktd6M6sNpJjZu865WcUdTuRI1DUkEmb+dNBPACOCziKSGxUCkeLxEtDRv8pIpETR7KMiIjFOLQIRkRinQiAiEuNUCEREYpwKgYhIjFMhEBGJcSoEIiIxToVARCTG/T/vSFsnz/zfNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr, tpr, umbrales = roc(y_test, y_pred)\n",
    "a=auc(y_test, y_pred)\n",
    "plt.plot(fpr, tpr)\n",
    "plt.plot(fpr, fpr, 'r--')\n",
    "plt.xlabel('FPR')\n",
    "plt.ylabel('TPR')\n",
    "plt.title('Binary ROC curve --- AUC={:.3f}'.format(a))\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd5wU9f3H8dfn7qiCSrPRETRgx7PFLopY0ViCGqMJyo8QjdHYEhMLMdiNDQv22LBEE5JgSDQxGmMBBRFQBJGOUqSIUq58fn9852RvuTvuuJud29338/HYx+3Mzs5+ZhfmM9/vd+Yz5u6IiEj+Kkg6ABERSZYSgYhInlMiEBHJc0oEIiJ5TolARCTPKRGIiOQ5JQKpNTO71syeTDoOEWlYSgQ5zsxmm9kaM1ttZl+Y2WNm1ipDn+1m9nX02QvM7HYzK0xb5ngzezdabpmZPWVmndKW2d7MHjazRWb2lZl9bGbXmdkWmdiOxiZKyG5m+1Uxf6NEHS3bM2X6aDN7Pfoul5jZf8zsxM2I40wzmxP9dn8ys7Y1LHuCmU2J/i38z8z6pL1+sZl9bmarzOwRM2uW8tqeZvaGma00s/lm9pu6xio1UyLIDye4eyugL1AM/Dp9AQvi+PewR/TZhwLfB36c8pmnAk8DdwDtgV2AdcB/zaxNtExb4C2gBXCAu7cGjgK2BnaMId6K2IriWnd9mJkBPwS+jP7W9f2nAs8DfwA6AdsCVwMn1HE9uwAPAGdH6/gGuLeaZXsBTwFDCb/bX4AxFd+xmR0NXAn0A7oCPYDrUlbxNPA60Jbw72jY5iQuqYG765HDD2A2cGTK9C3AX6PnrwG/A94E1gA9gR2AMYQdzUzg/JT3Xgu8ADwLfAW8T9jRV/fZDvRMmX4OGBk9N2AOcHnaewqAKcDwaPp64EOgoA7bfBDwP2AFMA84N2V7z0tZ7lzgv2nx/hSYAXwG3AfcmrbuPwOXRM93AP4ILImW/1kGfs9Dot/qLGAZ0DTt93myut8h+s7nApc1QBwjgKdTpncE1gOtq1j2AuBvab/xGqBfNP00MCLl9X7A5ynT3wB9UqafB36Z1P+pXHyoRZBHzKwzcCwwMWX22cAQoDVhxzwamE/YyZ0KjDCzI1KWH0j4j9iW8B/4T2bWpBaf/R3gYEJyAdgZ6BKt61vuXk7YuR4VzToSeDGaX5tt7Aq8DNwNdAD2BCbV5r2Rk4D9gD7AM8D3o6NwolZKf2B01Hr6C/AB0JGw8/p5dHQbp3Oiz30umq7LkfzOQGdCMq+SmR1kZitqeBwULboLYdsBcPdPCYlgp+pWnfbcgF2rWlf0fFszaxdN3wH80MyamNnOwAHAK7XZYKkdJYL88CczWwH8F/gP4WiuwmPuPtXdS4HtgAOBK9x9rbtPAh6ichfEe+7+gruXALcDzYH9a/js983sa+AjwhF5RfdB++jvoiresyjl9XbVLFOdM4FX3P0Zdy9x92XRdtTWDe7+pbuvAd4gHE0fHL12KvCWuy8E9gE6uPtwd1/v7rOAB4FBdfisOjGzlsBphCPxEsIOvS7dQxU71mq/T3f/r7tvXcPjv9GirYCVaW9fSTigSPcKcKiZHWZmTYFfAU2BltWsq+J5xbr+Svju1wAfAw+7+/hNbq3UmhJBfjgp+k/c1d2HRTu5CvNSnu8AfOnuX6XMm0M44t1o+egovaL1UJ2+hP/o3yccaVcM8C6N/m5fxXu2T3l9WTXLVKcz8Gkdlk+Xun1OaCGdEc06k9DXDaEve4fUo2XCDm7b9BWaWZdokHS1ma2O5r2cMu+s6FEx/XI1sZ0MlAJjo+mngGPMrEM0XQpUap2ltNZKCN8l1O37rM5qYMu0eVsSugwrcfePCS2Ze9iQ5KcR/u1Uta6K519FY0R/B4YTDjo6A0eb2bAG2AaJKBFIavnZhUBbM0s9qusCLEiZ7lzxJOoe6RS9r/oPCJ4jDPpeHc2eTtgRnJa6bLTOU4BXo1mvACfXYSB7HtUPIn/NhqNQCC2gjcJNm34GODXqctqP0G1V8TmfpR0tt3b3Yzdaoftcd29V8YjmHZMy76noUTF9TDXxn0NIqnPN7HNCt1oTQoKC0P/fLe093QkJYgHhO59H+H6rZGYHpyatKh4VraOpwB4p7+sBNAM+qWq9UStyV3dvB1wTxVlxVF9pXdHzL9x9GWHguMzd/+Dupe4+n5CcN/qepR6SHqTQI94HaYPFaa+9RsrgaTTvDcKRW3Ngd+CLivcTBiNLgO8BRcAl0fqbVLP+9MHi3Qg74+2i6e8Dqwg7suaEHfMjhB1au2iZttFnPAF0jeZ1JHRL7V7FZ3YhHJWeHsXYDtgzeu130Ta3JAyezmDjweKeVazzI+CfwEsp8woJg+VXEM5oKiT0ee8T0+/YESgjjFFsl/K4kdBdB7ANoVvlbEKCaEvoPhqdsp5To2V+RDjyLiAMro+qYzy7RL/dwYRW3pOpn1PF8ntH31EHwvhG6kDzAOBzwrjM1sC/gBuj17YkDPqfGcW6HeGAYkRd4tVjE79n0gHoEfMPXPdE0InQJ/sloYtlaMpr11L5rKGJQN8aPnujHSthIPe2lOmBhCPDr6PPfAbonPaeHQgJ4vPocz8mHFW2rOZzDwbeiXZU84BzovntgX9E63gz2p7aJILfRK+dVkVcz0RxLQferu67boDf8cqKHX4VMZQAu0bT3yWMBS0ntNQeAtqkvWcAIeGvJpzx9Bpw3GbEdCYhaX9NOJuqbdrv/KuU6f9G3/uXhNNOt0hb1yWEg45VwKNAs5TXjoj+jayMvusHq/vt9di8h0VftIiI5CmNEYiI5DklAhGRPKdEICKS55QIRETyXKMsrFWT9u3be7du3ZIOQ0Qkq7z33ntL3b1DVa9lXSLo1q0bEyZMSDoMEZGsYmZzqntNXUMiInlOiUBEJM8pEYiI5DklAhGRPKdEICKS52JLBNENqBeb2ZRqXjczu8vMZprZZDPrG1csIiJSvThbBI8RqhxW5xigV/QYQrg/rIiIZFhs1xG4++tm1q2GRQYCf/BQ/vRtM9vazLZ397rcllBEpNEoKSvn0Tc/Y/Xa0gZdb9Hab2i54kv2OXxv9ui8dYOuG5K9oKwjlW+TOD+at1EiMLMhhFYDXbp0yUhwIiJ19eGClYwY+zEAZg2zzgNmf8ANf7+br5ptwaQX/5lziaDW3H0UMAqguLhYN1AQkUapvDzsnp4YvC8H96qymkPtrVgBl10Gox+Cnj3hoYfY9bvdGyDKjSWZCBaQcv9bwp2xFlSzrIhI/igrg+9+F6ZPh8svh2uvhRYtYvu4JBPBGOACMxtNuCn4So0PiEi2cHfKyit3UJSW17PDYtkyaNsWCgvhd7+Dzp2huLh+66yF2BKBmT0DHAa0N7P5hHvMNgFw9/uBscCxwEzgG8LNtEVEssKVf/yQZyfMq/K1wroOELjDU0/BRRfBjTfC+efDySc3QJS1E+dZQ2ds4nUHfhrX54uIxGnW0tV0btuC0/fuXGl+y2ZF9O3apvYrmjcPhg6FsWNh//3hwAMbONJNy4rBYhGRpCxauYbLX5jMmvVlleZ//PlX7N5pKy7s12vzV/7MM/B//xfGBO64Ay64IHQLZZhKTIiI1GDawlW8MWMpJWXlNGtS8O1jj85bcdJeHeu38jZtYL/9YMqU0C2UQBIAtQhEJEcs/motT709l9Ly8gZd7+yl3wDw25N2ZfdO9TyHv7QUfv97WL8erroKBgyAo49uuIsONpMSgYjkhLGTF3HnqzMoLDAaerfadoumbLdl8/qt5IMPYPBgeO89OP30MEBslngSACUCEclCs5asZv7yNZXmzVi8GoD3f30UW7VskkRYVVu3Dq6/PpwN1LYtPP88nHJKo0gAFZQIRCTrnHLf/1j+TclG85sWFtC0qJENfc6YATfdBGeeCbffDu3aJR3RRpQIRCSjysudknr243+9voyT9tyBsw/oWml++1bNaNE0mQHXSlavhj//Gc46C3bdFT7+GHr0SDqqaikRiOQpd2d9WTlrS8pZV1rGupJy1paUsa40/K2YvzZ9fsWy0d+KZapc9tvn5ayLnq8va5jB3I5tWrB317YNsq4G9cor4YKwOXOgb1/o3btRJwFQIhBpFMrKPW3HWXnnmvq3Yie8tiT1edU74tQd9dr095aW4/WoiNC0qIDmRQU0a1JI8yYFNC8qpFn0t0WTQrZu0YTmTcK8ZkVhmYq/TQrr131TYMbAPXeo1zoa3PLlcOml8MgjsNNO8J//hCSQBZQIRFK4O+tKy2vegdaw811Xw868uiPptSVl9apRU1hgNC8qCDvdir/fPi+gzRZNv53/7c66SeG3O/FmFTvztHVUfr5hJ968SSFNCwsoKGg8g52JKysLVwR/8glccQVcc02sReIamhKBNFqlZeWsLa2uq2Hjnemmd76b7vZYV1q/bovqdpzNigpo1ayIdltsOGpuXsWRclU732YbzU95XlRAUT2PrqUeli7dUCRuxAjo0iV0B2UZJQLZpPLyir7k2vQFl7G2tLzSTjd1x53aLbFhfVUcSZeWb1TZsS6aFFqlnWz6zrd9q6Iqd67NqjsKrnQkvfH6QvdHAdaITgmUGLnDE0/Az38eTgsdMgROOinpqDabEkEWcXdKyrzaI9l1aQN5FQN0Gx9VVzM4WM1719fjKNmMKneczZuEnW7r5kV0qHSUW/POt+Yj6GhnrqNkidOcOaE+0Lhx4Z4BhxySdET1pkQQk4Ur1vDhgpUpR8O1ORtjw853XTVH2/Upd960sKDSzjS9G2LLFk0q7YArH0lX319cU1dG00IdJUsOefLJUCkU4O67YdgwKMj+gw4lgphc/Owk3vnsyypfKzA26utNHbDbukUTmrVuttFON33n26yK/uKqzuBo3qSQpkUFFGpwT6R+OnSAgw6CBx6Arl03vXyWUCKIydqSMoq7tuHGU3bfaCdeVGA6ShbJBiUlcOutoVjcb34TCsT179+oykM0BCWCGLVqXkTPbVolHYaIbI6JE0ORuIkTYdCgRlUkrqFlf+eWiEhDWrsWfvUr2GcfWLgQ/vjHcAOZHEwAFdQiqKe5y77hF89PYm1J5TNrZiz+iv17NL7iUiKyCTNnhu6gH/4Qbrst3DwmxykR1NO0RSsZP3s5+3ZrS6vmG77ODq2b8b2+9bx7kYhkxurV8NJLcPbZoUjc9OnQvXvSUWWMEkEDuW7gLvTefsukwxCRuho3LlwQNm8eFBeH+kB5lARAYwQikq+WLYNzzgm3i2zZEt54I2uKxDU0tQhEJP9UFImbOTPcO/jXv4bm9bwVZRZTIhCR/LFkSbhDWGFhuGtY166w555JR5U4dQ2JSO5zh0cfDfcJePDBMG/gQCWBiBKBiOS22bPDFcE//jHsthscfnjSETU6SgQikrueeCKcDvrWW3DvvfDaa6FVIJVojKAObnz5Y/7zyZJK81atKUkoGhHZpG23DWWi778/3DRGqqREUAd/n7KItSXl7NZpqw0z27TggB3b0b39FskFJiJBSQncfHM4K+jqq0OBuP79k46q0VMiqKP9erTlzkF7JR2GiKR7770wDjB5Mpx55oYicbJJGiMQkey2Zg1ceSXst184PfSll+Cpp5QE6iDWRGBmA8xsupnNNLMrq3i9i5n928wmmtlkMzs2znhEJMe4w6efwu23w7nnwrRpWX3v4KTElgjMrBAYCRwD9AHOMLM+aYv9GnjO3fcCBgH3xhWPiOSQVavgscfCUf+uu8KMGfDQQ7D11klHlpXibBHsC8x091nuvh4YDQxMW8aBikptWwELY4xHRHLB2LFh5z94MHz0UZiXQ7eNTEKciaAjMC9len40L9W1wA/MbD4wFriwqhWZ2RAzm2BmE5YsWVLVIiKS65YuDWWijzsOWreGN9/M2yJxDS3pweIzgMfcvRNwLPCEmW0Uk7uPcvdidy/u0KFDxoMUkYRVFIkbPTqcFvr++7D//klHlTPiPH10AdA5ZbpTNC/VYGAAgLu/ZWbNgfbA4hjj2qSpC1dy+QuTKSmrfNexBSvWsEdn9UGKZMwXX0CHDqFI3K23hi6g3XdPOqqcE2eLYDzQy8y6m1lTwmDwmLRl5gL9AMysN9AcSLzv58P5K5m6cBU7bN2CHTu0+vZxZO9tOaVvp6TDE8l97vDww7DzzjBqVJh3wglKAjGJrUXg7qVmdgEwDigEHnH3qWY2HJjg7mOAXwAPmtnFhIHjc93d44qprm743m5sv1WLpMMQyS+zZsH558O//gWHHgpHHpl0RDkv1iuL3X0sYRA4dd7VKc+nAQfGGYOIZJHHH4dhw0JX0P33h4RQkPRQZu5TiQkRaTx22AGOOALuuw86qRs2U5QIRCQ569fDjTdCeTlcey0cdVR4SEapzSUiyRg/HvbeG665JowLNJ7hwbyjRCAimfXNN3DppeE6gOXLYcwY+MMfVCQuQUoEIpJZn30Gd98N550HU6eG00IlURojEJH4rVwJL74IP/oR7LJLKBKnO4Y1GmoRiEi8/va3sPM/7zz4+OMwT0mgUVEiEJF4LFkCZ50Fxx8PbdqEG8h/5ztJRyVVUNeQiDS8sjI46KAwHnDddeEOYk2bJh2VVEOJQEQazuefwzbbhCuDb7sNunUL9w6QRk1dQyJSf+Xl8MADsNNO4S+ELiElgaygRCAi9TNzJvTrB0OHwj77wNFHJx2R1JESgYhsvkcfhd12CzeKefBBeOUV6NEj6aikjjRGICKbr0uX0AIYORI6pt+JVrKFEoGI1N66dXDDDWFMYPjw0CXUr1/SUUk9qWtIRGrnnXdCkbjrroO5c1UkLocoEYhIzb7+Gi65BA44IJSK+Otf4bHHVCQuhygRiEjN5syBe+8NZwVNnQrHHZd0RNLANEYgIhtbsQJeeCHUB+rTJ5wiqjuG5Sy1CESksj//Oez8hw7dUCROSSCnKRGISLB4MQwaBCedBB06wNtvq0hcnlDXkIiEInEHHhjOBrr+erj8cmjSJOmoJEOUCETy2cKFsN12oUjcnXeGInF9+iQdlWSYuoZE8lF5Odx3X+j6uf/+MO/YY5UE8pQSgUi++eQTOPxwGDYM9tsPjjkm6YgkYUoEIvnk4Ydhjz1g8mR45BH4xz+ge/eko5KEaYxAJJ906xZaACNHwvbbJx2NNBJ5nwgmzl3OjMWrK80bP3t5QtGINLB16+C3vw3Pr79eReKkSnmfCIY++R5frFq30fxmRQVs0Szvvx7JZv/7HwweHC4K+/GPQ5E41QeSKuT9nm59aTnf26sjl/TfqdL81s2bsGVznUctWWj1arjqKrj7bujcGf7+d901TGoU62CxmQ0ws+lmNtPMrqxmmdPNbJqZTTWzp+OMpzqtmhfRqU3LSo+tWigJSJaaOzfcN/inP4UpU5QEZJNiaxGYWSEwEjgKmA+MN7Mx7j4tZZlewC+BA919uZltE1c8Ijlt+XJ4/nkYMiRcCzBrFuywQ9JRSZaIs0WwLzDT3We5+3pgNDAwbZnzgZHuvhzA3RfHGI9IbnrppbDzHzYMpk8P85QEpA7iTAQdgXkp0/Ojeal2AnYyszfN7G0zG1DVisxsiJlNMLMJS5YsiSlckSzz+edw2mnwve+FMhHvvgs775x0VJKFkh4sLgJ6AYcBnYDXzWw3d1+RupC7jwJGARQXF+v+eCJlZXDwwTBvHowYAZdeqiJxstniTAQLgM4p052ieanmA++4ewnwmZl9QkgM42OMSyR7zZ8fun0KC+Guu8JVwSoVLfUUZ9fQeKCXmXU3s6bAIGBM2jJ/IrQGMLP2hK6iWTHGJJKdysvD6aDf+U4oFgfhCmElAWkAsSUCdy8FLgDGAR8Bz7n7VDMbbmYnRouNA5aZ2TTg38Bl7r4srphEstLHH8Mhh8DPfgYHHQTHH590RJJjYh0jcPexwNi0eVenPHfgkughIukeegguuABatoTHH4ezz9bVwdLgkh4sFpGa7LgjnHAC3HMPbLtt0tFIjlIiEGlM1q6F4cPD8xEjwn0DDj882Zgk5+l+BCKNxZtvwp57wg03wJIloUicSAYoEYgk7auv4MILw3UB69bBuHHw4IMaC5CMUSIQSdr8+WFQ+MIL4cMPoX//pCOSPKMxApEkLFsGzz0HP/kJ9O4disTpjmGSkDq3CMyswMzOiiMYkZznDi+8EIrE/exnG4rEKQlIgqpNBGa2pZn90szuMbP+FlxIuPL39MyFKJIjFi2CU04JheI6d4YJE1QkThqFmrqGngCWA28B5wG/Agw4yd0nZSA2kdxRUSRuwQK4+Wa4+GIoUs+sNA41/Uvs4e67AZjZQ8AioIu7r81IZCK5YN486NgxFIkbOTIUidtpp02/TySDahojKKl44u5lwHwlAZFaKisL1UFTi8QdfbSSgDRKNbUI9jCzVYTuIIAWKdPu7lvGHp1INvroIxg8GN56K1QIPeGEpCMSqVG1icDdCzMZiEhOGDUqXA/QujU88QScdZYuDJNGr9pEYGbNgaFAT2Ay8EhUWlpEqtOrF5x8cugW2mabpKMRqZWauoYeJ4wTvAEcC+wCXJSJoESyxpo1cO214aj/xhtVJE6yUk2JoE/KWUMPA+9mJiSRLPH663DeeTBjBgwdGi4WUzeQZKHanjWkLiGRCqtWwbBhcOih4eygV18NZwYpCUiWqqlFsGd0lhCEM4V01pAIwMKF8NhjcMkl4d4BW2yRdEQi9VJTIvjA3ffKWCQijdnSpaFI3LBh4dqAzz7THcMkZ9TUNaS7Yoi4w7PPhiJxP/85fPJJmK8kIDmkphbBNmZW7U3l3f32GOIRaTwWLgxloseMgeLiMBagK4MlB9WUCAqBVmy4slgkf5SVwSGHhCJxt94KF12kInGSs2r6l73I3YdnLBKRxmDOHOjUKRSJu/de6NEDevZMOiqRWNU0RqCWgOSPsjK4/fZwt7CKInH9+ysJSF6oqUXQL2NRiCRpypRQJO7dd+H44+Gkk5KOSCSjqm0RuPuXmQxEJBH33w99+4Z7Bj/9dBgY7tQp6ahEMqrO9ywWyQkenR3du3e4deS0aXDGGbo6WPKSToOQ/PLNN3D11WEw+KabQpmIQw9NOiqRRKlFIPnjtddg993htttg9eoNrQKRPKdEILlv5Ur4v//bUB76X/8K9w9WN5AIoEQg+WDRInjySbj0Upg8WfcLEEkTayIwswFmNt3MZprZlTUsd4qZuZkVxxmP5JElS+Duu8Pz73wHZs+GW26Bli0TDUukMYotEZhZITASOAboA5xhZn2qWK414c5n78QVi+QR93AaaO/e8ItfbCgS16FDsnGJNGJxtgj2BWa6+yx3Xw+MBgZWsdxvgZuAtTHGIvlg3jw44YRww/iePWHiRBWJE6mFOBNBR2BeyvT8aN63zKwv0Nnd/1bTisxsiJlNMLMJS5YsafhIJfuVlsJhh8G//w2//z28+SbsskvSUYlkhcSuIzCzAuB24NxNLevuo4BRAMXFxTrnTzaYPRs6dw6VQR94IBSJ69Ej6ahEskqcLYIFQOeU6U7RvAqtgV2B18xsNrA/MEYDxlIrpaWhPHTv3qFKKMCRRyoJiGyGOFsE44FeZtadkAAGAWdWvOjuK4H2FdNm9hpwqbtPiDEmyQWTJ4cicRMmwMCBcMopSUckktViSwTuXmpmFwDjCDe5ecTdp5rZcGCCu4+J67Or8upHX3DR6EmUlpdXmr+2pJwCXViUPe69N9wkpk2bcAvJ007ThWEi9RTrGIG7jwXGps27upplD4szlpmLV7N6XSk/OrAbTQtTesQMTu2rapONnnvY4e+6KwwaFAaE27ff9PtEZJPyrujcZUfvTMumebfZ2evrr+HXvw6DwbfcEm4fecghSUclklNUYkIar1dfhd12gzvugHXrVCROJCZKBNL4rFgB550XzgIqKoLXX4e77tJYgEhMlAik8fniCxg9Gq64Aj74AA4+OOmIRHKaOsulcajY+V90Eey8c7hQTIPBIhmhFoEkyz2UiO7TBy6/HGbMCPOVBEQyRolAkjN3Lhx3HJx9dmgFTJoEvXolHZVI3lHXkCSjokjc4sVhIHjYsHAfYRHJOCUCyaxZs6Br13A20IMPwo47QrduSUclktfUNSSZUVoKN90UxgJGjgzz+vVTEhBpBNQikPhNmhSKxL3/Ppx8cqgPJCKNhloEEq977oF99oEFC+CFF+DFF2H77ZOOSkRSKBFIPCrKQey+e7h15LRpKhct0kipa0ga1urVcNVV0KRJuHGMisSJNHpqEUjD+cc/Qpnou++GkhIViRPJEkoEUn/Ll8OPfgRHHw3Nm4cicXfeqSJxIllCiUDqb/HiMBD8y1+GM4QOOijpiESkDjRGIJvn88/hmWfg4os3FIlr1y7pqERkM6hFIHXjDo8/Hi4M++UvNxSJUxIQyVpKBFJ7s2fDgAFw7rkhEahInEhOUNeQ1E5pKRx+OCxdGkpEDB0KBTqOEMkFSgRSs5kzoXv3UCTukUegR49QNE5EcoYO6aRqJSUwYgTsssuGInGHH64kIJKD1CKQjb3/figSN2lSKBD3/e8nHZGIxEgtAqnsrrtg333D6aEvvgjPPQfbbpt0VCISIyUCCSrKQey1F/zwh6FI3MknJxuTiGSEuoby3VdfhesBmjWD226Dgw8ODxHJG2oR5LO//z0Uibv33tAiUJE4kbykRJCPli2Dc86BY46BLbaAN9+E229XkTiRPKVEkI+WLYOXXoLf/AYmToQDDkg6IhFJUKyJwMwGmNl0M5tpZldW8folZjbNzCab2atmppPU47JoUbhRjDvstBPMmQPDh4exARHJa7ElAjMrBEYCxwB9gDPMrE/aYhOBYnffHXgBuDmuePKWe7giuHfv0AKYOTPMb9Mm2bhEpNGIs0WwLzDT3We5+3pgNDAwdQF3/7e7fxNNvg10ijGe/PPZZ9C/f7g4bI894IMPVCRORDYS5+mjHYF5KdPzgf1qWH4w8HJVL5jZEGAIQJcuXRoqvtxWWgpHHBHGA+67D4YMUZE4EalSo7iOwMx+ABQDh1b1uruPAkYBFBcX6xzHmsyYEQrDFRXBo4/CjjtC585JRyUijVich13IJYcAAAnxSURBVIgLgNQ9UKdoXiVmdiRwFXCiu6+LMZ7cVlIC118frgu4554w77DDlAREZJPibBGMB3qZWXdCAhgEnJm6gJntBTwADHD3xTHGktsmTAjjAJMnw6BBcMYZSUckIlkkthaBu5cCFwDjgI+A59x9qpkNN7MTo8VuAVoBz5vZJDMbE1c8OevOO2G//cINY/7853Af4W22SToqEckisY4RuPtYYGzavKtTnh8Z5+fnNPdwJXBxcWgN3HwzbL110lGJSBZqFIPFUgerVsEVV0Dz5vD738OBB4aHiMhm0vmE2WTs2HDHsFGjwllBKhInIg1AiSAbLF0KP/gBHHccbLUV/O9/cMstKhInIg1CiSAbLF8Of/kLXHNNuI3kfjVdlyciUjcaI2isFiyAp56Cyy4LZSHmzNFgsIjEQi2CxsYdHnwQ+vSBa6+FTz8N85UERCQmSgSNyaefQr9+oS5Q377hArGePZOOSkRynLqGGovS0pAEvvwSHngAzjtPReJEJCOUCJI2fXooDFdUBI8/Hp53UjVuEckcHXImZf16uO462G03GDkyzDv0UCUBEck4tQiS8O67oSzElClw5plw1llJRyQieUwtgky7445ws/iKawOeegrat086KhHJY0oEmVJRDmLffeH882HqVDj++GRjEhFBXUPxW7kSLr8cWrQIrYHvfjc8REQaCbUI4vSXv4QLwx56KFQLVZE4EWmElAjisGRJuEvYiSdCu3bwzjtw440qEicijZISQRxWroSXXw6nh06YEG4eIyLSSGmMoKHMmwdPPglXXhnKQsyZE0pGi4g0cmoR1Fd5Odx/f7hhzPXXbygSpyQgIllCiaA+ZsyAI46An/wknBb64YcqEiciWUddQ5urtBSOOgpWrAhnBf34xxoMFpGspERQVx99FG4UU1QETzwRisTtsEPSUYmIbDZ1DdXWunVw9dWw++5wzz1h3sEHKwmISNZTi6A23n47FImbNi3cRP7ss5OOSESkwahFsCm33RZKQnz1FYwdG7qD2rVLOioRkQajRFCd8vLw94ADYOjQUDL6mGOSjUlEJAbqGkq3YgVcemkoEnf33SoSJyI5Ty2CVH/6UygS99hj0KqVisSJSF5QIgBYvBhOPx1OPhm22SYUibvhBl0XICJ5QYkAYNUq+Oc/4Xe/g/HjYe+9k45IRCRj8neMYO7ccAbQr34VykLMnQutWycdlYhIxsXaIjCzAWY23cxmmtmVVbzezMyejV5/x8y6xRkPEM4GuvfeUCRuxIgNReKUBEQkT8WWCMysEBgJHAP0Ac4wsz5piw0Glrt7T+D3wE1xxQPQY9l8mh3VD37603Ba6NSpKhInInkvzhbBvsBMd5/l7uuB0cDAtGUGAo9Hz18A+pnFM0JrpaX84bmrKZgyBR59FMaNg27d4vgoEZGsEmci6AjMS5meH82rchl3LwVWAhtdtmtmQ8xsgplNWLJkyWYF0227rXj2ohGs++BDOPdcnREkIhLJisFidx8FjAIoLi7erJP7+++yHf2HD27QuEREckGcLYIFQOeU6U7RvCqXMbMiYCtgWYwxiYhImjgTwXigl5l1N7OmwCBgTNoyY4BzouenAv9y1+W8IiKZFFvXkLuXmtkFwDigEHjE3aea2XBggruPAR4GnjCzmcCXhGQhIiIZFOsYgbuPBcamzbs65fla4LQ4YxARkZqpxISISJ5TIhARyXNKBCIieU6JQEQkz1m2na1pZkuAOZv59vbA0gYMJxtom/ODtjk/1Gebu7p7h6peyLpEUB9mNsHdi5OOI5O0zflB25wf4tpmdQ2JiOQ5JQIRkTyXb4lgVNIBJEDbnB+0zfkhlm3OqzECERHZWL61CEREJI0SgYhInsvJRGBmA8xsupnNNLMrq3i9mZk9G73+jpl1y3yUDasW23yJmU0zs8lm9qqZdU0izoa0qW1OWe4UM3Mzy/pTDWuzzWZ2evRbTzWzpzMdY0Orxb/tLmb2bzObGP37PjaJOBuKmT1iZovNbEo1r5uZ3RV9H5PNrG+9P9Tdc+pBKHn9KdADaAp8APRJW2YYcH/0fBDwbNJxZ2CbDwdaRs9/kg/bHC3XGngdeBsoTjruDPzOvYCJQJtoepuk487ANo8CfhI97wPMTjruem7zIUBfYEo1rx8LvAwYsD/wTn0/MxdbBPsCM919lruvB0YDA9OWGQg8Hj1/AehnltU3Md7kNrv7v939m2jybcId47JZbX5ngN8CNwFrMxlcTGqzzecDI919OYC7L85wjA2tNtvswJbR862AhRmMr8G5++uE+7NUZyDwBw/eBrY2s+3r85m5mAg6AvNSpudH86pcxt1LgZVAu4xEF4/abHOqwYQjimy2yW2Omsyd3f1vmQwsRrX5nXcCdjKzN83sbTMbkLHo4lGbbb4W+IGZzSfc/+TCzISWmLr+f9+krLh5vTQcM/sBUAwcmnQscTKzAuB24NyEQ8m0IkL30GGEVt/rZrabu69INKp4nQE85u63mdkBhLse7uru5UkHli1ysUWwAOicMt0pmlflMmZWRGhOLstIdPGozTZjZkcCVwEnuvu6DMUWl01tc2tgV+A1M5tN6Esdk+UDxrX5necDY9y9xN0/Az4hJIZsVZttHgw8B+DubwHNCcXZclWt/r/XRS4mgvFALzPrbmZNCYPBY9KWGQOcEz0/FfiXR6MwWWqT22xmewEPEJJAtvcbwya22d1Xunt7d+/m7t0I4yInuvuEZMJtELX5t/0nQmsAM2tP6CqalckgG1httnku0A/AzHoTEsGSjEaZWWOAH0ZnD+0PrHT3RfVZYc51Dbl7qZldAIwjnHHwiLtPNbPhwAR3HwM8TGg+ziQMygxKLuL6q+U23wK0Ap6PxsXnuvuJiQVdT7Xc5pxSy20eB/Q3s2lAGXCZu2dta7eW2/wL4EEzu5gwcHxuNh/YmdkzhGTePhr3uAZoAuDu9xPGQY4FZgLfAD+q92dm8fclIiINIBe7hkREpA6UCERE8pwSgYhInlMiEBHJc0oEIiJ5TolApJbMrMzMJqU8upnZYWa2Mpr+yMyuiZZNnf+xmd2adPwi1cm56whEYrTG3fdMnRGVMH/D3Y83sy2ASWb2l+jlivktgIlm9pK7v5nZkEU2TS0CkQbi7l8D7wE90+avASZRz8JgInFRIhCpvRYp3UIvpb9oZu0INY2mps1vQ6j383pmwhSpG3UNidTeRl1DkYPNbCJQDtwYlUA4LJr/ASEJ3OHun2cwVpFaUyIQqb833P346uabWXfgbTN7zt0nZTo4kU1R15BIzKJy0DcCVyQdi0hVlAhEMuN+4JDoLCORRkXVR0VE8pxaBCIieU6JQEQkzykRiIjkOSUCEZE8p0QgIpLnlAhERPKcEoGISJ77fz5FOWdTuqtRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr2, tpr2, umbrales2 = roc(y_test, y_prob)\n",
    "a2=auc(y_test, y_prob)\n",
    "plt.plot(fpr2, tpr2)\n",
    "plt.plot(fpr2, fpr2, 'r--')\n",
    "plt.xlabel('FPR')\n",
    "plt.ylabel('TPR')\n",
    "plt.title('Prob ROC curve --- AUC={:.3f}'.format(a2))\n",
    "plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian Gridsearching\n",
    "\n",
    "## HyperOpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "\n",
    "from hyperopt import fmin, hp, tpe, Trials, space_eval, STATUS_OK\n",
    "\n",
    "from hyperopt.pyll import scope as ho_scope\n",
    "from hyperopt.pyll.stochastic import sample as ho_sample\n",
    "\n",
    "from sklearn.metrics import mean_squared_error as mse\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carat</th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>depth</th>\n",
       "      <th>table</th>\n",
       "      <th>price</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.21</td>\n",
       "      <td>Premium</td>\n",
       "      <td>J</td>\n",
       "      <td>VS2</td>\n",
       "      <td>62.4</td>\n",
       "      <td>58.0</td>\n",
       "      <td>4268</td>\n",
       "      <td>6.83</td>\n",
       "      <td>6.79</td>\n",
       "      <td>4.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.32</td>\n",
       "      <td>Very Good</td>\n",
       "      <td>H</td>\n",
       "      <td>VS2</td>\n",
       "      <td>63.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>505</td>\n",
       "      <td>4.35</td>\n",
       "      <td>4.38</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.71</td>\n",
       "      <td>Fair</td>\n",
       "      <td>G</td>\n",
       "      <td>VS1</td>\n",
       "      <td>65.5</td>\n",
       "      <td>55.0</td>\n",
       "      <td>2686</td>\n",
       "      <td>5.62</td>\n",
       "      <td>5.53</td>\n",
       "      <td>3.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.41</td>\n",
       "      <td>Good</td>\n",
       "      <td>D</td>\n",
       "      <td>SI1</td>\n",
       "      <td>63.8</td>\n",
       "      <td>56.0</td>\n",
       "      <td>738</td>\n",
       "      <td>4.68</td>\n",
       "      <td>4.72</td>\n",
       "      <td>3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.02</td>\n",
       "      <td>Ideal</td>\n",
       "      <td>G</td>\n",
       "      <td>SI1</td>\n",
       "      <td>60.5</td>\n",
       "      <td>59.0</td>\n",
       "      <td>4882</td>\n",
       "      <td>6.55</td>\n",
       "      <td>6.51</td>\n",
       "      <td>3.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   carat        cut color clarity  depth  table  price     x     y     z\n",
       "0   1.21    Premium     J     VS2   62.4   58.0   4268  6.83  6.79  4.25\n",
       "1   0.32  Very Good     H     VS2   63.0   57.0    505  4.35  4.38  2.75\n",
       "2   0.71       Fair     G     VS1   65.5   55.0   2686  5.62  5.53  3.65\n",
       "3   0.41       Good     D     SI1   63.8   56.0    738  4.68  4.72  3.00\n",
       "4   1.02      Ideal     G     SI1   60.5   59.0   4882  6.55  6.51  3.95"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv('data/diamonds_train.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df.drop('price', axis=1)\n",
    "y=df.price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "clarity={'I1':0, 'SI2':1, 'SI1':2, 'VS2':3, 'VS1':4, 'VVS2':5, 'VVS1':6, 'IF':7}\n",
    "\n",
    "cut={'Fair':0, 'Good':1, 'Very Good':2, 'Premium':3, 'Ideal':4}\n",
    "\n",
    "color={'J':0, 'I':1, 'H':2, 'G':3, 'F':4, 'E':5, 'D':6}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def labeling(s, dic):\n",
    "    return dic[s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carat</th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>depth</th>\n",
       "      <th>table</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.21</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>62.4</td>\n",
       "      <td>58.0</td>\n",
       "      <td>6.83</td>\n",
       "      <td>6.79</td>\n",
       "      <td>4.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.32</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>63.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>4.35</td>\n",
       "      <td>4.38</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>65.5</td>\n",
       "      <td>55.0</td>\n",
       "      <td>5.62</td>\n",
       "      <td>5.53</td>\n",
       "      <td>3.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.41</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>63.8</td>\n",
       "      <td>56.0</td>\n",
       "      <td>4.68</td>\n",
       "      <td>4.72</td>\n",
       "      <td>3.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.02</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>60.5</td>\n",
       "      <td>59.0</td>\n",
       "      <td>6.55</td>\n",
       "      <td>6.51</td>\n",
       "      <td>3.95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   carat  cut  color  clarity  depth  table     x     y     z\n",
       "0   1.21    3      0        3   62.4   58.0  6.83  6.79  4.25\n",
       "1   0.32    2      2        3   63.0   57.0  4.35  4.38  2.75\n",
       "2   0.71    0      3        4   65.5   55.0  5.62  5.53  3.65\n",
       "3   0.41    1      6        2   63.8   56.0  4.68  4.72  3.00\n",
       "4   1.02    4      3        2   60.5   59.0  6.55  6.51  3.95"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.clarity=X.clarity.apply(lambda x: labeling(x, clarity))\n",
    "\n",
    "X.cut=X.cut.apply(lambda x: labeling(x, cut))\n",
    "\n",
    "X.color=X.color.apply(lambda x: labeling(x, color))\n",
    "\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test=train_test_split(X,y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "space={\n",
    "    'n_estimators':hp.quniform('n_estimators', 10, 1000, 25),\n",
    "    'learning_rate':hp.uniform('learning_rate', 0.0001, 1.0),\n",
    "    'max_depth':hp.quniform('x_max_depth', 4, 16, 1),\n",
    "    'min_child_weight':hp.quniform('x_min_child', 1, 10, 1),\n",
    "    'subsample':hp.uniform('x_subsample', 0.7, 1),\n",
    "    'gamma':hp.uniform('x_gamma', 0.1, 0.5),\n",
    "    'reg_lambda':hp.uniform('x_reg_lambda', 0, 1)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objetivo(space):\n",
    "    \n",
    "    modelo=xgb.XGBRegressor(\n",
    "        n_estimators=int(space['n_estimators']),\n",
    "        learning_rate=space['learning_rate'],\n",
    "        max_depth=int(space['max_depth']),\n",
    "        min_child_weight=space['min_child_weight'],\n",
    "        subsample=space['subsample'],\n",
    "        gamma=space['gamma'],\n",
    "        reg_lambda=space['reg_lambda'],\n",
    "        objective='reg:squarederror'\n",
    "    )\n",
    "    \n",
    "    eval_set=[(X_train, y_train), (X_test, y_test)]\n",
    "    \n",
    "    modelo.fit(X_train, y_train, eval_set=eval_set, eval_metric='rmse', verbose=False)\n",
    "    \n",
    "    y_pred=modelo.predict(X_test)\n",
    "    \n",
    "    rmse=mse(y_test, y_pred)**0.5\n",
    "    \n",
    "    return {'loss':rmse, 'status':STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [02:42<00:00, 16.29s/it, best loss: 513.2538682139791]\n",
      "{'learning_rate': 0.05529808016919341, 'n_estimators': 200.0, 'x_gamma': 0.3239970072610857, 'x_max_depth': 12.0, 'x_min_child': 9.0, 'x_reg_lambda': 0.6148663551256083, 'x_subsample': 0.7055054119072453}\n"
     ]
    }
   ],
   "source": [
    "best=fmin(fn=objetivo, space=space, algo=tpe.suggest, max_evals=10, trials=Trials())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learning_rate': 0.05529808016919341, 'n_estimators': 200.0, 'x_gamma': 0.3239970072610857, 'x_max_depth': 12.0, 'x_min_child': 9.0, 'x_reg_lambda': 0.6148663551256083, 'x_subsample': 0.7055054119072453}\n"
     ]
    }
   ],
   "source": [
    "print (best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelo=xgb.XGBRegressor(\n",
    "    n_estimators=int(best['n_estimators']),\n",
    "    learning_rate=best['learning_rate'],\n",
    "    x_max_depth=int(best['x_max_depth']),\n",
    "    x_min_child=best['x_min_child'],\n",
    "    x_subsample=best['x_subsample'],\n",
    "    x_gamma=best['x_gamma'],\n",
    "    x_reg_lambda=best['x_reg_lambda'],\n",
    "    objective='reg:squarederror'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "592.8827773384666"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelo.fit(X_train, y_train)\n",
    "y_pred=modelo.predict(X_test)\n",
    "mse(y_test, y_pred)**0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[12:02:11] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "602.8909646176038"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m=xgb.XGBRegressor()\n",
    "m.fit(X_train, y_train)\n",
    "y_pred=m.predict(X_test)\n",
    "mse(y_test, y_pred)**0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}