logistic regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Odds Ration** = p/(1-p)\n",
    "\n",
    "logit function = log - odds\n",
    "logit(P) = log p/(1-p)\n",
    "\n",
    "sigmoid function is the inverse form of the logit function, which is used to predict the probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-7.00000000e+00, -6.90000000e+00, -6.80000000e+00, -6.70000000e+00,\n",
       "       -6.60000000e+00, -6.50000000e+00, -6.40000000e+00, -6.30000000e+00,\n",
       "       -6.20000000e+00, -6.10000000e+00, -6.00000000e+00, -5.90000000e+00,\n",
       "       -5.80000000e+00, -5.70000000e+00, -5.60000000e+00, -5.50000000e+00,\n",
       "       -5.40000000e+00, -5.30000000e+00, -5.20000000e+00, -5.10000000e+00,\n",
       "       -5.00000000e+00, -4.90000000e+00, -4.80000000e+00, -4.70000000e+00,\n",
       "       -4.60000000e+00, -4.50000000e+00, -4.40000000e+00, -4.30000000e+00,\n",
       "       -4.20000000e+00, -4.10000000e+00, -4.00000000e+00, -3.90000000e+00,\n",
       "       -3.80000000e+00, -3.70000000e+00, -3.60000000e+00, -3.50000000e+00,\n",
       "       -3.40000000e+00, -3.30000000e+00, -3.20000000e+00, -3.10000000e+00,\n",
       "       -3.00000000e+00, -2.90000000e+00, -2.80000000e+00, -2.70000000e+00,\n",
       "       -2.60000000e+00, -2.50000000e+00, -2.40000000e+00, -2.30000000e+00,\n",
       "       -2.20000000e+00, -2.10000000e+00, -2.00000000e+00, -1.90000000e+00,\n",
       "       -1.80000000e+00, -1.70000000e+00, -1.60000000e+00, -1.50000000e+00,\n",
       "       -1.40000000e+00, -1.30000000e+00, -1.20000000e+00, -1.10000000e+00,\n",
       "       -1.00000000e+00, -9.00000000e-01, -8.00000000e-01, -7.00000000e-01,\n",
       "       -6.00000000e-01, -5.00000000e-01, -4.00000000e-01, -3.00000000e-01,\n",
       "       -2.00000000e-01, -1.00000000e-01, -2.48689958e-14,  1.00000000e-01,\n",
       "        2.00000000e-01,  3.00000000e-01,  4.00000000e-01,  5.00000000e-01,\n",
       "        6.00000000e-01,  7.00000000e-01,  8.00000000e-01,  9.00000000e-01,\n",
       "        1.00000000e+00,  1.10000000e+00,  1.20000000e+00,  1.30000000e+00,\n",
       "        1.40000000e+00,  1.50000000e+00,  1.60000000e+00,  1.70000000e+00,\n",
       "        1.80000000e+00,  1.90000000e+00,  2.00000000e+00,  2.10000000e+00,\n",
       "        2.20000000e+00,  2.30000000e+00,  2.40000000e+00,  2.50000000e+00,\n",
       "        2.60000000e+00,  2.70000000e+00,  2.80000000e+00,  2.90000000e+00,\n",
       "        3.00000000e+00,  3.10000000e+00,  3.20000000e+00,  3.30000000e+00,\n",
       "        3.40000000e+00,  3.50000000e+00,  3.60000000e+00,  3.70000000e+00,\n",
       "        3.80000000e+00,  3.90000000e+00,  4.00000000e+00,  4.10000000e+00,\n",
       "        4.20000000e+00,  4.30000000e+00,  4.40000000e+00,  4.50000000e+00,\n",
       "        4.60000000e+00,  4.70000000e+00,  4.80000000e+00,  4.90000000e+00,\n",
       "        5.00000000e+00,  5.10000000e+00,  5.20000000e+00,  5.30000000e+00,\n",
       "        5.40000000e+00,  5.50000000e+00,  5.60000000e+00,  5.70000000e+00,\n",
       "        5.80000000e+00,  5.90000000e+00,  6.00000000e+00,  6.10000000e+00,\n",
       "        6.20000000e+00,  6.30000000e+00,  6.40000000e+00,  6.50000000e+00,\n",
       "        6.60000000e+00,  6.70000000e+00,  6.80000000e+00,  6.90000000e+00])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# plot the sigmoid function for some values\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def sigmoid(z):\n",
    "    return 1.0/(1.0 + np.exp(-z))\n",
    "\n",
    "z = np.arange(-7, 7, 0.1)\n",
    "z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.11051194e-04, 1.00677082e-03, 1.11253603e-03, 1.22939862e-03,\n",
       "       1.35851995e-03, 1.50118226e-03, 1.65880108e-03, 1.83293894e-03,\n",
       "       2.02532039e-03, 2.23784852e-03, 2.47262316e-03, 2.73196076e-03,\n",
       "       3.01841632e-03, 3.33480731e-03, 3.68423990e-03, 4.07013772e-03,\n",
       "       4.49627316e-03, 4.96680165e-03, 5.48629890e-03, 6.05980149e-03,\n",
       "       6.69285092e-03, 7.39154134e-03, 8.16257115e-03, 9.01329865e-03,\n",
       "       9.95180187e-03, 1.09869426e-02, 1.21284350e-02, 1.33869178e-02,\n",
       "       1.47740317e-02, 1.63024994e-02, 1.79862100e-02, 1.98403057e-02,\n",
       "       2.18812709e-02, 2.41270214e-02, 2.65969936e-02, 2.93122308e-02,\n",
       "       3.22954647e-02, 3.55711893e-02, 3.91657228e-02, 4.31072549e-02,\n",
       "       4.74258732e-02, 5.21535631e-02, 5.73241759e-02, 6.29733561e-02,\n",
       "       6.91384203e-02, 7.58581800e-02, 8.31726965e-02, 9.11229610e-02,\n",
       "       9.97504891e-02, 1.09096821e-01, 1.19202922e-01, 1.30108474e-01,\n",
       "       1.41851065e-01, 1.54465265e-01, 1.67981615e-01, 1.82425524e-01,\n",
       "       1.97816111e-01, 2.14165017e-01, 2.31475217e-01, 2.49739894e-01,\n",
       "       2.68941421e-01, 2.89050497e-01, 3.10025519e-01, 3.31812228e-01,\n",
       "       3.54343694e-01, 3.77540669e-01, 4.01312340e-01, 4.25557483e-01,\n",
       "       4.50166003e-01, 4.75020813e-01, 5.00000000e-01, 5.24979187e-01,\n",
       "       5.49833997e-01, 5.74442517e-01, 5.98687660e-01, 6.22459331e-01,\n",
       "       6.45656306e-01, 6.68187772e-01, 6.89974481e-01, 7.10949503e-01,\n",
       "       7.31058579e-01, 7.50260106e-01, 7.68524783e-01, 7.85834983e-01,\n",
       "       8.02183889e-01, 8.17574476e-01, 8.32018385e-01, 8.45534735e-01,\n",
       "       8.58148935e-01, 8.69891526e-01, 8.80797078e-01, 8.90903179e-01,\n",
       "       9.00249511e-01, 9.08877039e-01, 9.16827304e-01, 9.24141820e-01,\n",
       "       9.30861580e-01, 9.37026644e-01, 9.42675824e-01, 9.47846437e-01,\n",
       "       9.52574127e-01, 9.56892745e-01, 9.60834277e-01, 9.64428811e-01,\n",
       "       9.67704535e-01, 9.70687769e-01, 9.73403006e-01, 9.75872979e-01,\n",
       "       9.78118729e-01, 9.80159694e-01, 9.82013790e-01, 9.83697501e-01,\n",
       "       9.85225968e-01, 9.86613082e-01, 9.87871565e-01, 9.89013057e-01,\n",
       "       9.90048198e-01, 9.90986701e-01, 9.91837429e-01, 9.92608459e-01,\n",
       "       9.93307149e-01, 9.93940199e-01, 9.94513701e-01, 9.95033198e-01,\n",
       "       9.95503727e-01, 9.95929862e-01, 9.96315760e-01, 9.96665193e-01,\n",
       "       9.96981584e-01, 9.97268039e-01, 9.97527377e-01, 9.97762151e-01,\n",
       "       9.97974680e-01, 9.98167061e-01, 9.98341199e-01, 9.98498818e-01,\n",
       "       9.98641480e-01, 9.98770601e-01, 9.98887464e-01, 9.98993229e-01])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phi_z = sigmoid(z)\n",
    "phi_z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEKCAYAAADn+anLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHoRJREFUeJzt3Xl8VPW9//HXZyb7TgiELRL2TVAwoEVbwX2rW+2t/mrFpddWRb0ubbX9tfW2t1prry11/bn1utSlLq1oqStgvbUooCyGnbCFhDUkIXtm5vv7I6MGZCABkjPL+/l4zGO2k8k7eZzkPd9z5nyPOecQERHZF5/XAUREJHqpJEREJCKVhIiIRKSSEBGRiFQSIiISkUpCREQiUkmIiEhEKgkREYlIJSEiIhEleR3gUBUUFLji4mKvY4js28qVbdcjRnibQ2QvCxcu3OGc63Wg5WK+JIqLi1mwYIHXMUT2bcqUtuu5c71MIfIlZrahI8tpc5OIiESkkhARkYhUEiIiEpFKQkREIlJJiIhIRCoJERGJSCUhIiIRqSRERCQilYSIiESkkhARkYhUEiIiEpFKQkREIlJJiIhIRN1WEmb2hJltM7NPIzxvZvYHM1tjZkvMbEJ3ZRMRkX3rzpHE/wBn7Of5M4Fh4cvVwEPdkElERPaj284n4Zz7h5kV72eR84CnnHMOmGdmeWbW1zlX2S0BRUSiiHOOkINgyBFybZdgyBEKQTB8PzMlifQUf5fmiKaTDvUHNrW7Xx5+7EslYWZX0zbaoLCwkLk6oYtEqaOrqwFYpHU0ZoScozEATYG268aAoyngaAlBSxBaQo7W8HVLsO2x1s9uh58LOgiG/5kHQuH77R4LhiDQ/n74dsCBcxBy4DqQddroFKYekdylv49oKokOc849AjwCUFJS4qZ8dvYvkWiTlweA1lFvNLUG2b67mZ31LVTVN7OzroWq+rbLzvoWdoWvdze1UtccoK4pQH1LsFPfI8XvIzXZR1qyn7RkH6lJfpL9PpJTjFSfken3kew3knxfXCf5rW0Zv5Hk95Hsa7tO8ht+M/w+wxe+9vsMM770uM9nHDson+GF2V3022sTTSWxGShqd39A+DERkX2qbw6wbkc963bUU1HdSGVNE5urGz+/XVXfss+vS/H7yM9MIT8zhZ5ZKfTPSyc7LYms1CSywtfZaUlkpyWTlZpEZqo/XAJ+0pP9exSC32fd/FN3r2gqiZnAdDN7HjgWqNH+CBEB2FXfQmlFLWu27Wbt9nrKdtSxdls9W2qb9lguOy2Jfrnp9M1L46iiPPrlptE7O62tELJS6BkuhqzUJMzi+5/74dJtJWFmzwFTgAIzKwd+DiQDOOceBmYBZwFrgAbgiu7KJiLRo745wKJN1SzcsItPN9dQWlHL5urGz5/PTktiSK8sJg/tyZBeWQzplUlxQWZ4NNC12+cTUXd+uumSAzzvgOu6KY6IRIn65gAfrN3JvLKdzF9fRWlFLcGQwwwGFWQyYWAPLvvKQMb0y2VEn2wKslI0CuhG0bS5SUQSxOqtu5mzchtzV25n/voqWoOO1CQfRxflcc2JQygp7sGEgT3I0cjAcyoJEekWZdvreH1JJa8trmD1tjoAhhdmccXxg5gyvBfHFPcgNalrP/MvnaeSEJEuU9PYyisfl/PSwnJKK2oBmFSczy/OG8Mpowrpl5fucUI5EJWEiBx2S8treGbeBmYurqCxNcjY/rn837NHcfa4vvTNVTHEEpWEiBwWzjlmr9jG/XPW8MnGatKT/Zw/vh/fPnYgR/bP9TqeHCSVhIgcklDI8dayrdw3ezWlFbUM6JHOHV8fzYXHDNCO5zigkhCRg/bequ3cNWs5K7bsZlBBJvdcNI7zx/cn2a9T1cQLlYSIdNq6HfX81+vLeHfFNgYVZDLj4qM5Z1y/uJ+iIhGpJESkw+qaA9z37mqe+Oc6UpP8/PiskVw+eRApSRo5xCuVhIh0yAdrd/CDF5dQUdPIN48ZwK2nj6B3dprXsaSLqSREZL+aWoPc/cYK/vjP9QwuyOTlayYz4YgeXseSbqKSEJGIFm2q5uY/L6Jsez2XTy7mR2eM7PIzoUl0UUmIyJc453hm3gb+87Vl9MpO5U/fPZbjhxZ4HUs8oJIQkT00B4L87K+lvLBgEyeN7M3vvnU0uek63iFRqSRE5HNba5v4/jML+WRjNdOnDuXmU4fj08daE5pKQkSAtvmWrnpyPnXNAR789gTOGtvX60gSBVQSIsK8sp1898kF5KYn88q1kxnZJ8frSBIlVBIiCW72iq1c88zHFOVn8MxVx9InV8c+yBdUEiIJ7LXFFdz0wiJG9c3hySsnkZ+Z4nUkiTIqCZEE9cL8jdz2ylImFufz+LQSsjVjq+yDSkIkAc1cXMFtryzlxOG9ePjSY0hL1gFysm+alUskwcxduY2bX1jExOJ8FYQckEpCJIEs3FDF959ZyIg+2Tw2rUQFIQekkhBJEMsra7nij/Ppm5vOk1dO0lnjpENUEiIJYHN1I5c98REZKUk8fdUkCrJSvY4kMUI7rkXiXGNLkO89vYDGliCvXDuZAT0yvI4kMUQlIRLHnHPc9soSSitqeeyyEoYXZnsdSWKMNjeJxLFH/lHGq4squPW0EZw8qtDrOBKDVBIicWruym3c/cYKzh7bl2unDPE6jsQolYRIHFq3o54bnvuE4YXZ3PPNcZhpum85OCoJkTjTHAgy/dmP8fmMRy8rISNFux7l4GntEYkzv31zJaUVtTx6WQlF+fokkxwajSRE4sj7q7fz6Pvr+M5xAzl1tHZUy6FTSYjEiZ11zdz858UM653FT84e5XUciRPa3CQSB5xz/PClJdQ0tvLUlZM0J5McNhpJiMSBp+dt4N0V27j9zJGM6qtTj8rho5IQiXHrd9Rz56zlTBnRi8snF3sdR+KMSkIkhjnnuP2VpST7fPz6Qh0PIYefSkIkhr0wfxP/KtvJ7WeNok9umtdxJA6pJERi1NbaJn41aznHDsrn4olFXseROKWSEIlRP3v1U1oCIX79jXH4fNrMJF1DJSESg/6+tJI3S7dy06nDGVSQ6XUciWMqCZEYU9PQys9mlnJk/xy+e8Igr+NInNPBdCIx5t63V7Kzrpk/Xj6RJL/e50nX0homEkOWV9by9LwNXHrcQI7sn+t1HEkAKgmRGOGc4+czS8lNT+bmU4d7HUcShEpCJEa8vqSSj9ZVcevpI8jLSPE6jiQIlYRIDGhoCXDnrOWM6ZfDxROP8DqOJBDtuBaJAQ/MWUNlTRP3XTIev46JkG6kkYRIlFu/o55H/7GOC8b3p6Q43+s4kmBUEiJR7s5Zy0n2G7edOdLrKJKAVBIiUWz++ireWraVa6YMoTBHE/hJ91NJiEQp5xx3zlpOYU4qV50w2Os4kqBUEiJR6s3SLXyysZqbThlOeopORyreUEmIRKHWYIi731jJsN5ZXHTMAK/jSAJTSYhEoefnb2Ldjnp+dMZIzc8kntLaJxJl6poDzHhnFZMG5XPyqN5ex5EEp4PpRKLMo/8oY0ddC49NG6VzVovnNJIQiSI76pp59P0yzh7bl6OL8ryOI6KSEIkmD81dS1NrkFtO0yyvEh1UEiJRYktNE0/P28A3JgxgcK8sr+OIACoJkahx/5zVhEKOG04e5nUUkc+pJESiwKaqBl6Yv4lvTSyiKD/D6zgin1NJiESB+2avxsyYftJQr6OI7EElIeKxsu11vPzxZi49diB9c9O9jiOyB5WEiMdmvLuaFL+Pa6YM8TqKyJeoJEQ8tGrrbmYurmDa5GJ6Zad6HUfkS1QSIh6a8e5qMpL9fO9rmgpcopNKQsQja7btZtbSSi6bXEyPzBSv44jsk0pCxCMPzFlLWpKf754wyOsoIhGpJEQ8sH5HPa8u2sy3jz2CnlnaFyHRSyUh4oEH564hye/jau2LkCinkhDpZpuqGnjl481cMrGI3jlpXscR2S+VhEg3e/i9tZjB907UcRES/VQSIt1oS00TLy4o56JjiuiXp6OrJfqpJES60cPvrSXoHNfq6GqJESoJkW6ybXcTz320kQvG99dMrxIzOl0SZpZpZv6uCCMSzx57fx2twRDXTdVMrxI7DlgSZuYzs/9jZn8zs23ACqDSzJaZ2T1mpjVe5ACq6lt4Zt4Gvn5UPwYVZHodR6TDOjKSmAMMAW4H+jjnipxzvYETgHnA3WZ2aRdmFIl5j/9vGY2tQaZrFCExJqkDy5zinGs1s4uApZ896JyrAl4GXjaz5K4KKBLrAiHHkx9s4Kwj+zKsMNvrOCKdcsCRhHOuNXzzaeDZ9vsjzOyKvZbZLzM7w8xWmtkaM7ttH89fbmbbzWxR+PLdjv0YItFrS00Tdc0BnXVOYlJndlyvAN5jz5HD9R394nC5PACcCYwGLjGz0ftY9AXn3NHhy2OdyCcSdQIhR2VNI6eOLmRU3xyv44h0WmdKwjnnHgZeAWaaWTpgnfj6ScAa51yZc64FeB44rxNfLxJzttY2EQw5bjhpmNdRRA5KZ0piF4Bz7ingceBvQGc+7N0f2NTufnn4sb19w8yWmNlLZlbUidcXiSr1zQEqa5rIy0hh7IBcr+OIHJSO7LgGwDl3crvbL5lZE/A/hznPa8BzzrlmM/se8CRw0t4LmdnVwNUAhYWFzJ079zDHEDl0f1/XygXBELlJAa2jErMOWBJmZs45t/fjzrnXgYL9LbOXzUD7kcGA8GPtX3Nnu7uPAb/Z1ws55x4BHgEoKSlxU6ZMOcC3FuleTa1Bbv3fOUxLT6ZvQQ59tY5KjOrQcRJmdr2ZHdH+QTNLMbOTzOxJYFoHXmc+MMzMBplZCnAxMHOv1+zb7u65wPIOvK5I1Hnuo43sqGtmQA9N4iexrSObm84ArgSeM7PBtO2bSKetYN4Cfu+c++RAL+KcC5jZdOBNwA884ZwrNbNfAAucczOBG8zsXCAAVAGXH8TPJOKp5kCQ//deGZOK88lJ0yFEEtsOWBLOuSbgQeBBM8sGsoEG51x1Z7+Zc24WMGuvx37W7vbttB3ZLRKzXlxQzpbaJu755ri2z/CJxLAOf7rJzG4A1gMfAf8ys+u6KpRIrGoJhHho7lrGH5HHCUMLvI4jcsg6MsHfDDObBvwHMMo5NwD4GjDGzH7Z1QFFYslfPilnc3UjN5w8DLPOHEYkEp06M8FfAfCBmX0M3AOsBS42sx5dmE8kZrQGQ9w/Zw3jBuQyZXgvr+OIHBYd2SfxV+CvZnYccBNQCYwDjgLygdlmluOc06m2JKG9uqiCTVWN/PycMRpFSNzo8MF0wHXAn4FFtM0GOwpY6pybEv5Iq0jCCgRDPDBnDaP75nDyqN5exxE5bDq849o5txo4FngJSAOWABeEn2vpknQiMeL1JZWs21GvfRESdzozkvisDP4WvogIEAw57pu9mpF9sjltdKHXcUQOq06f41pE9jRraSVrt9dz/UnD8Pk0ipD4opIQOQSh8ChiWO8szjyyj9dxRA47lYTIIXizdAurttYx/aShGkVIXFJJiBykUMgx493VDC7I5Jxx/byOI9IlVBIiB+md5VtZsWU3008ail+jCIlTKgmRg+Cc4w+zVzOwZwbnHqVRhMQvlYTIQXhr2VY+3VzLdVOHkuTXn5HEL63dIp0UCjnufWsVgwsyuXD8vk7TLhI/VBIinfS3pZWs3LqbG08ZplGExD2t4SKdEAiG+N07qxhRmM3X9YkmSQAqCZFO+OuiCsq213PTqcN1XIQkBJWESAe1BkPMeHcVR/bP4fQxmqNJEoNKQqSDXlxQzqaqRm45dYRmepWEoZIQ6YCm1iD3zV7NhCPymDJCZ52TxKGSEOmAZ+ZtoLKmiVtO0yhCEotKQuQAahpbuX/OGr46rIDjhxZ4HUekW6kkRA7goblrqWls5bYzR3odRaTbqSRE9qOiupEn/rmO84/uz5h+uV7HEel2KgmR/bj37VXg4JbThnsdRcQTKgmRCJZX1vLyx+VMmzyQAT0yvI4j4gmVhEgEv/77CrJTk7hu6lCvo4h4RiUhsg//XLOD91Zt57qpQ8nLSPE6johnVBIiewmGHL/623L65aYxbXKx13FEPKWSENnLsx9tZFllLbefNYq0ZL/XcUQ8pZIQaWdXfQv//dZKjhuczznj+nodR8RzKgmRdv777ZXUNrZyx7ljNP2GCCoJkc+VVtTw7Icb+c5xAxnZJ8frOCJRQSUhAjjn+M+Zy8jLSOHmU0d4HUckaqgkRICZiyv4aH0VPzh9BLkZyV7HEYkaKglJeHXNAe6atYKx/XP5t5Iir+OIRJUkrwOIeO03b6xg6+4mHrx0An6dt1pkDxpJSEKbv76Kp+dt4PLJxUw4oofXcUSijkpCElZTa5AfvbyEfrnp3HqadlaL7Is2N0nCun/2Gsq21/PUlZPITNWfgsi+aCQhCWlZRS0Pv7eWCyf052vDe3kdRyRqqSQk4QSCIX708hJy05P56dmjvY4jEtU0xpaE88j7ZSzdXMN9l4ynR6amARfZH40kJKEs3lTNvW+t4qyxfTSBn0gHqCQkYdQ3B7jx+U/onZ3KXReM0wR+Ih2gzU2SMO6YWcqGqgae//fjNPWGSAdpJCEJ4bXFFby4sJzpU4dy7OCeXscRiRkqCYl75bsa+PFflnJ0UR43nDzM6zgiMUUlIXGtJRDixucX4Rz84eLxJPu1yot0hvZJSFy747VSFm7YxX2XjOeInhlexxGJOXpbJXHrmXkbePbDjXz/xCF8/ah+XscRiUkqCYlLH62r4o6ZpUwZ0YsfnK7J+0QOlkpC4k5FdSPX/mkhR+RnMOPi8TpHhMghUElIXGlsCXL10wtobg3xyGUl5KbreAiRQ6Ed1xI3AsEQ1z/3MaUVtTx2WQlDe2d5HUkk5mkkIXHBOcdPX/2Ud5Zv4xfnjuHkUYVeRxKJCyoJiQv3vr2K5z7axHVTh/CdrxR7HUckbqgkJOY9OHcN981ewyWTinQaUpHDTCUhMe2x98v4zRsrOf/ofvzX+WM1s6vIYaYd1xKzHpq7lrvfWMFZY/vw228epY+6inQBlYTEHOccM95dze/fWc25R/Xj3n87iiTNySTSJVQSElOCIcfPZ37KM/M2ctExA7j7G+M0ghDpQioJiRmNLUFuemERb5Ru4XsnDua2M0ZqH4RIF1NJSEzYUtPEvz+1gE8ravjpOaO56oRBXkcSSQgqCYl689dXMf3Zj6lrCvDod0o4ZbQOlBPpLioJiVqhkOOh99Zy79urGNAjnSevnMTIPjlexxJJKCoJiUrbdzdz858X8f7qHZwzri93XTiW7DRN1ifS3VQSEnXeW7WdW19cTG1jK3deMJZLJhVpB7WIR1QSEjV21DXzy9eX8eqiCob0yuSpKycxqq82L4l4SSUhnnPO8eKCcn41azkNLQFuPHkY104dQmqS3+toIglPJSGe+mTjLu6atYKP1lcxsbgHd104lqG9s72OJSJhKgnxxJptu7nnzZW8WbqVnpkp3HXhWL5VUoRPR0+LRBWVhHSrdTvqeWjuGl5aWE5GShI3nzqcK08YRFaqVkWRaKS/TOlyzjnmr9/Fo++X8c7yrST7fFxx/CCunTKEnlmpXscTkf1QSUiXaWgJ8ManW3jyg/UsLq8hLyOZ66YM5bLJA+mdneZ1PBHpAJWEHFafjRpeWriJWUu3UNccYFBBJr88/0i+MaE/GSla5URiif5i5ZAFQ46FG3bx9rItvFG6hU1VjWSk+Dl7bF8uOmYAE4vztUNaJEapJOSg7Kxr5l9lO/nHqu28u3wbO+tbSPH7+MqQntx0ynDOOLKPRg0icUB/xdIh22qb+GRTNR+WVfHB2h2s2LIbgOzUJKaO7M1pYwo5cXgvza8kEmdUErIH5xzb65pZUbmb5ZW1LC6vZtHGaipqmgBITfJRUtyDH5w+gq8M6cm4/rk6dahIHFNJJKim1iCbqhrYsLOB9Tvr2bCzgbXb61ixZTdV9S2fLzegRzoTBvbgyqI8xh+Rx5h+uaQla7oMkUTRrSVhZmcAMwA/8Jhz7td7PZ8KPAUcA+wEvuWcW9+dGeNBcyDIjroWttQ0sbW27bKltomtNU1U1jSxqaqBytomnPvia7LTkhjcK4tTRxUysm82I/pkM7JPDvmZKd79ICLiuW4rCTPzAw8ApwLlwHwzm+mcW9ZusauAXc65oWZ2MXA38K3uyhgNnHM0tYaobwnQ0BykrjlAQ0uA+pYgDc0B6sKXXQ2tVDe0sKuhlV31LexqaKG6oZVdDS00tAS/9Lopfh+9c1Lpk5PGcYN7MrBnJsUFGQzsmcnA/AzyMpI1HbeIfEl3jiQmAWucc2UAZvY8cB7QviTOA+4I334JuN/MzLn273kPj8qaRjbubCDoHMFQ2yXkHIFg23UwBIFQ6PPboZAjEHIEnfv8dih8/7Ov/+wSCDlaAiFagkGaW0O0BENt9wNtt5sDe95vCYRoDgRpaA5S3xIg1IGf1gxy0pLJz0whLyOZwpw0RvTJpkdGCj0ykumZ1VYInxVDfmaKSkBEOq07S6I/sKnd/XLg2EjLOOcCZlYD9AR2HO4wMxdVcNffVxzul8VnkOTzkZLUdkkNX6f4v3gsxe8jJz2ZFP8Xz6cm+chISSIr1U9GahKZKX4yUpLITPWTmZr0xe2UJDJTk8hNT8avYw9EpIvF5I5rM7sauBqgsLCQuXPndvo1ejaG+OHENHzW9o/dB/h84WsDn9kXzxn4w9dm4N/ruc+/3tjPu3UHBMOXA3BAc/iyu+0rdocvEluOrq4GYNFBrKMi0aA7S2IzUNTu/oDwY/taptzMkoBc2nZg78E59wjwCEBJSYmbMmVKV+QVOXR5eQBoHZVY1Z0fcJ8PDDOzQWaWAlwMzNxrmZnAtPDti4DZXbE/QkREOqbbRhLhfQzTgTdp+wjsE865UjP7BbDAOTcTeBx42szWAFW0FYmIiHikW/dJOOdmAbP2euxn7W43Ad/szkwiIhKZ5lMQEZGIVBIiIhKRSkJERCJSSYiISEQqCRERiUglISIiEakkREQkIpWEiIhEpJIQEZGIVBIiIhKRSkJERCJSSYiISEQqCRERichi/XQNZrYd2OB1jr0U0AWnXO1Cytu1YilvLGUF5T0UA51zvQ60UMyXRDQyswXOuRKvc3SU8natWMobS1lBebuDNjeJiEhEKgkREYlIJdE1HvE6QCcpb9eKpbyxlBWUt8tpn4SIiESkkYSIiESkkuhCZna9ma0ws1Iz+43XeTrCzG4xM2dmBV5n2R8zuyf8u11iZn8xszyvM+3NzM4ws5VmtsbMbvM6z/6YWZGZzTGzZeH19UavMx2ImfnN7BMze93rLB1hZnlm9lJ4vV1uZl/xOlNHqCS6iJlNBc4DjnLOjQF+63GkAzKzIuA0YKPXWTrgbeBI59w4YBVwu8d59mBmfuAB4ExgNHCJmY32NtV+BYBbnHOjgeOA66I8L8CNwHKvQ3TCDOAN59xI4ChiJLtKoutcA/zaOdcM4Jzb5nGejvgd8EMg6ndUOefecs4FwnfnAQO8zLMPk4A1zrky51wL8DxtbxqiknOu0jn3cfj2btr+gfX3NlVkZjYAOBt4zOssHWFmucDXgMcBnHMtzrlqb1N1jEqi6wwHvmpmH5rZe2Y20etA+2Nm5wGbnXOLvc5yEK4E/u51iL30Bza1u19OFP/Tbc/MioHxwIfeJtmv39P2hibkdZAOGgRsB/4Y3kT2mJlleh2qI5K8DhDLzOwdoM8+nvoJbb/bfNqG7hOBP5vZYOfhx8kOkPfHtG1qihr7y+ucezW8zE9o21Typ+7MFq/MLAt4GfgP51yt13n2xczOAbY55xaa2RSv83RQEjABuN4596GZzQBuA37qbawDU0kcAufcKZGeM7NrgFfCpfCRmYVom7dle3fl21ukvGY2lrZ3OovNDNo23XxsZpOcc1u6MeIe9vf7BTCzy4FzgJO9LN8INgNF7e4PCD8WtcwsmbaC+JNz7hWv8+zH8cC5ZnYWkAbkmNkzzrlLPc61P+VAuXPus9HZS7SVRNTT5qau81dgKoCZDQdSiJ6JvfbgnFvqnOvtnCt2zhXTtkJP8LIgDsTMzqBtc8O5zrkGr/Psw3xgmJkNMrMU4GJgpseZIrK2dwePA8udc/d6nWd/nHO3O+cGhNfVi4HZUV4QhP+WNpnZiPBDJwPLPIzUYRpJdJ0ngCfM7FOgBZgWhe92Y9n9QCrwdnj0M885931vI33BORcws+nAm4AfeMI5V+pxrP05HvgOsNTMFoUf+7FzbpaHmeLN9cCfwm8ayoArPM7TITriWkREItLmJhERiUglISIiEakkREQkIpWEiIhEpJIQEZGIVBIiIhKRSkJERCJSSYgcZmb2fTNbFL6sM7M5XmcSOVg6mE6ki4TnQpoN/MY595rXeUQOhkYSIl1nBm3zCqkgJGZp7iaRLhCeoXYgMN3jKCKHRJubRA4zMzsGeBL4qnNul9d5RA6FNjeJHH7TaTvh1JzwzuuYOMWmyL5oJCEiIhFpJCEiIhGpJEREJCKVhIiIRKSSEBGRiFQSIiISkUpCREQiUkmIiEhEKgkREYno/wOV3puwx/cbmQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(z, phi_z)\n",
    "plt.axvline(x=0, color = 'r')\n",
    "plt.ylim(-0.1,1.1)\n",
    "plt.xlabel('z')\n",
    "plt.ylabel('$\\phi (z)$')\n",
    "\n",
    "plt.yticks([0, 0.5, 1]) #set tick locations and labels on y axis\n",
    "ax = plt.gca()\n",
    "ax.yaxis.grid(True)\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic cost function is the sum-squared-error cost function. Use likelihood, log-likehood to maximize it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEOCAYAAACAfcAXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd81dX9+PHXyd57ELJJIBACYYShCCiKe1C1jiKO2lr1W/tt/X47/da2tv1922+X2lqte9AqxUmValVkVBkCsmeAhIQkZJK97/n9cW5CUMZNyL2fO97Px+PzuOtzP/fNx3jenzM+5yitNUIIIXyPn9UBCCGEsIYkACGE8FGSAIQQwkdJAhBCCB8lCUAIIXyUJAAhhPBRkgCEEMJHSQIQQggfJQlACCF8VIDVAQwUF5+gW4NiSY0JJS48yOpwhBDCPbVUQ9MRSClk0+bParXWiUM5jFMTgFKqBGgGeoEerXXR6fbPzMyk4eKH+NEV4/ja7FHODE0IITzXyl/Byv+FBz9F+QeUDvUwrqgBXKC1rnVkRz8/BUB7V69TAxJCCI/W1QoBIeDnf1aHcas+AAUE+itaJQEIIcSpdbdBYNhZH8bZCUAD/1JKbVJK3eXIF8KCAmjr6nFyWEII4cG6WiEo4qwP4+wmoPO01keUUknA+0qpPVrr1QN3sCeGuwAyMjLICA2guUMSgBDO1N3dTXl5OR0dHVaHIhwUEhJCWloagYGB0NEIIdFnfUynJgCt9RH7Y7VS6g1gOrD6c/s8CTwJUFRUpKNCAmlq73ZmWEL4vPLyciIjI8nKykIpZXU44gy01tTV1VFeXk52djZ0NA1LAnBaE5BSKlwpFdn3HLgY2HGm70WGBNDUIQlACGfq6OggPj5eCn8PoZQiPj7+eI2toxFCos76uM7sA0gG/q2U2gpsAN7RWr97pi9FhQRKE5AQLiCFv2c54b+XuzcBaa0PAoWD/V5UqDQBCSHEaXUOTwJwq2Gg0NcEJDUAIcTpbdmyheXLlw/puytXruTKK68c5ojOzgMPPEB6ejoREWcY3WOzmT6AYPduAhqSqJBAWjp76LXJYvVCiFM7mwTgjq666io2bNhw5h27mgHtnTWAqNBAAFqkFiCEV3vxxReZOHEihYWFLFq0CICSkhLmzZvHxIkTufDCCzl8+DAAS5cupaCggMLCQubMmUNXVxcPPvggS5YsYdKkSSxZsuSEY19xxRVs27YNgMmTJ/PQQw8B8OCDD/LUU08B0NLSwvXXX8/YsWNZuHAhWpuLzk2bNjF37lymTp3KJZdcQmVlJQDnn38+3//+95k+fTpjxoxhzZo1X/g33Xrrrbz55pv9rxcuXMhbb73l0PmYOXMmKSkpZ96xo8k8unMfwFBFhpiQmjq6iQ4LtDgaIbzfz/6xk10VTcN6zPyRUfzkqvGn/Hznzp384he/4JNPPiEhIYH6+noA7rvvPm677TZuu+02nn32Wb71rW/x5ptv8tBDD/Hee++RmprKsWPHCAoK4qGHHmLjxo386U9/+sLxZ8+ezZo1a8jMzCQgIICPP/4YgDVr1vDEE09QWVnJZ599xs6dOxk5ciSzZs3i448/ZsaMGdx333289dZbJCYmsmTJEh544AGeffZZAHp6etiwYQPLly/nZz/7GR988MEJv3vnnXfyhz/8gQULFtDY2Mgnn3zCCy+8wN69e7nxxhtPei5WrlxJTEyM4ye3o9E8DsMoILdLAFEhptCXoaBCeK8VK1bw5S9/mYSEBADi4uIAWLt2La+//joAixYt4nvf+x4As2bN4vbbb+eGG27g2muvPePxZ8+ezaOPPkp2djZXXHEF77//Pm1tbRw6dIi8vDwqKyuZPn06aWlpAEyaNImSkhJiYmLYsWMH8+fPB6C3t/eEq/K+3546dSolJSVf+N25c+dy7733UlNTw2uvvcZ1111HQEAAeXl5bNmyZYhn63P6E4AX1gCiQu01gHZpAhLCFU53pe4unnjiCdavX88777zD1KlT2bRp02n3nzZtGhs3bmTUqFHMnz+f2tpannrqKaZOndq/T3BwcP9zf39/enp60Fozfvx41q5de9Lj9n2nb/+TufXWW1m8eDGvvPIKzz33HICTagDe2AcgNQAhvN68efNYunQpdXV1AP1NQOeeey6vvPIKAH/961+ZPXs2AAcOHGDGjBk89NBDJCYmUlZWRmRkJM3NzSc9flBQEOnp6SxdupRzzjmH2bNn89vf/pY5c+acNq68vDxqamr6E0B3dzc7d+4c1L/t9ttv5+GHHwYgPz+//7hbtmw56Taowh+g095c542jgKLtncCNbZIAhPBW48eP54EHHmDu3LkUFhZy//33A/DHP/6R5557jokTJ/LSSy/xyCOPAPDd736XCRMmUFBQwLnnnkthYSEXXHABu3btOmknMJhmoKSkJEJDQ5k9ezbl5eX9CeVUgoKCePXVV/n+979PYWEhkyZN4pNPPhnUvy05OZlx48Zxxx13DOp73/ve90hLS6OtrY20tDR++tOfnnzHNpMsCY0d1PFPRvX1fLuDoqIivfqTdeQ/+B7fv3Qs95yfY3VIQnil3bt3M27cOKvD8EptbW1MmDCBzZs3Ex199s00A+3evZtxFUvh3w/Dj2vBzw+l1KYzLbZ1Km5XAwgLCiAk0I/61k6rQxFCiEH54IMPGDduHPfdd9+wF/79WmshLB78zr74drtOYID48GDqWrusDkMIIQbloosuorR0yCs0OqatDsIThuVQblcDAIiPCKKuRRKAEEJ8QWuNqQEMA7dMAHHhQdRLDUAIIb6otda7awCSAIQQ4hTaaiHMixNAfHgQddIJLIQQJ9La3Ajm3TWAYDq6bbI4vBDilLxtOuhNmzYxYcIEcnNz+da3vsVJh+hrm3n05j6A+PAgAOkIFkKckrdNB33PPffw1FNPsX//fvbv38+7755kAUXdax69uwZgEoD0AwjhvWQ66OMqKytpampi5syZKKW+cJx+fTWA8MQzHtMR7nkfQIQkACFc5p8/gKrtw3vMERPgsl+d8mOZDvq4lStXcuTIkf6ZSQHS0tI4cuTIF3e29TUBDU8NwD0TQLiZca+2RTqChfBGMh30EA1zE5BbJoCkKJMAqpslAQjhdKe5UncX3j4ddGpqKuXl5f3vlZeXk5qa+sWdbb3gFwihcaf99zvKLfsAQgL9iQ4NpKqxw+pQhBBOINNBnzgddEpKClFRUaxbtw6tNS+++CLXXHPNFw+ueyFyxLDMAwRumgAARkSFUNUkCUAIbyTTQX/Rn//8Z772ta+Rm5tLTk4Ol1122Rd3svVCpAPrBjvI7aaD3rhxIwC3PruBY21dLPvmeRZHJYT3kemgncep00GvX8G40ufhhhf73/Oq6aD7jIgK5qjUAIQQHsTp00HrXogcOWyHc8tOYDBNQDXNnfT02gjwd9s8JYQQ/Zw6HbSt19wHEDV8TUBuW7ImR4dg01ArdwML4RTu1Pwrzkz3dAF6WGsAbpsARkSFAEhHsBBOEBISQl1dnSQBD6G1pq6ulpDGgxDlA01AKdGhAFQca2dSeozF0QjhXdLS0igvL6empsbqUISDQjprSdv8a5i+ctiO6bYJIC3OJICy+jaLIxHC+wQGBpKdnW11GGIwVvwCupsgOu3M+zrIbZuAokICiQ4NpKxBEoAQQtBQClFp4B84bId02wQAkB4XSll9u9VhCCGE9Y6VQmzmsB7SvRNAbBjlUgMQQgg4dhhiMob1kO6dAOLCKG9ol5EKQgjf1t0BzZUQ42E1AKWUv1LqM6XU24P9blpsKJ09NmpkVlAhhC9rLDOPHtgE9J/A7qF8MT02DEA6goUQvq3BfnexJzUBKaXSgCuAp4fy/fQ4kwAOy1BQIYQvazhkHj2sCehh4HuA7VQ7KKXuUkptVEpt/PxNKRlxYfj7KQ7WtDo5TCGEcGN1xRAYPqx3AYMTE4BS6kqgWmt92qV7tNZPaq2LtNZFiYknLnQcFOBHZlwYB2panBWmEEK4v9p9kDAalBrWwzqzBjALuFopVQK8AsxTSi0e7EFGJYZzoFpqAEIIH1a7HxLGDPthnZYAtNY/1Fqnaa2zgJuAFVrrWwZ7nJzECA7VttJrk6GgQggf1NVqRgF5UgIYLjmJEXT12uSGMCGEb6orNo8Jo4f90C5JAFrrlVrrK4fy3ZykcADpBxBC+Kba/ebRF2sAoxIiAKQfQAjhm2r3gfKDuFHDfmi3TwCx4UHEhwdRXC01ACGED6rdZ24ACwwZ9kO7fQIAGJMcyZ6qJqvDEEII16vaAUnjnXJoj0gA40dGsaeqmZ7eU95PJoQQ3qezxXQCp0x0yuE9IgHkj4yis8fGwVrpBxBC+JDqXYCGEROccniPSADjR0YDsLOi0eJIhBDChSq3mscRPlwDyEkMJyjAj10V0g8ghPAhVdshNHZY1wEeyCMSQIC/H2NHRLJTEoAQwpdUbTPNP8M8B1Afj0gAYDqCd1Y0yepgQgjf0NsNR3c5rfkHPCgB5KdE0djeTUVjh9WhCCGE89Xuh95OSQAAE9JiANhadsziSIQQwgWO2GfSHznJaT/hMQkgPyWK4AA/NpY0WB2KEEI4X9k60wEcP/yTwPXxmAQQFOBHYXoMm0rrrQ5FCCGc7/B6SJ8Bfs4rpj0mAQAUZcays6KJ9q5eq0MRQgjnaa2Duv0mATiRZyWArFh6bJot0g8ghPBmZevNY8ZMp/6MRyWAKRmxANIMJITwbmXrwC8QRk526s94VAKICQtidFIEG0ulI1gI4cUOrzejfwJDnfozHpUAwDQDbS5twCZrBAshvFFPJ1R85vT2f/DEBJAZR1NHD3uqmq0ORQghht+RTeYGMCe3/4MHJoDzRicAsHp/jcWRCCGEExR/CMofsmY7/ac8LgEkR4UwdkQkq/ZKAhBCeKHiDyB9OoTGOP2nPC4BAMwdk8jG0npaO3usDkUIIYZPSw1UboGcC13ycx6bALp7NWsP1FkdihBCDJ+DH5nHXEkApzQ1K5bQQH/pBxBCeJfiDyEsHlKcNwHcQB6ZAIID/Dk3J55V+yQBCCG8hM0GBz6EnHlOnf9nII9MAABz8xIprWujRBaKF0J4g6pt0FoDuRe57Cc9NgHMGZ0IwMq91RZHIoQQw6D4ffOYM89lP+mxCSArIZzRSRG8u7PK6lCEEOLs7f4HpE2DiCSX/aTHJgCASwtGsOFQPXUtnVaHIoQQQ9dQCpVbYdxVLv1Zj08ANg3v7zpqdShCCDF0u/9hHiUBOC4/JYqMuDCW75BmICGEB9v1FiRPgLhRLv1Zj04ASimumJjCx8W11EozkBDCEzWUQPkGKPiSy3/aoxMAwIJJqfTaNG9vrbA6FCGEGLztS83jhC+7/KedlgCUUiFKqQ1Kqa1KqZ1KqZ8543fyRkQydkQkb26RBCCE8DBaw7alkHEOxGS4/OedWQPoBOZprQuBScClSimnTHC9YHIqW8qOcUhuChNCeJKqbVC715Krf3AgASil3ldKxQx4HauUeu9M39NGi/1loH1zyjJeCyal4qdg6cYyZxxeCCGcY/NL4B8M413f/g+O1QAStNbH+l5orRsAh+5UUEr5K6W2ANXA+1rr9UML8/RGRIdwQV4SSzeV09Nrc8ZPCCHE8Opuh21/h/yrISzOkhAcSQA2pVR/45RSKhMHr+S11r1a60lAGjBdKVXw+X2UUncppTYqpTbW1Ax9crcbp6VT09zJR7JQjBDCE+xaBp2NMOVWy0JwJAE8APxbKfWSUmoxsBr44WB+xF6D+Ai49CSfPam1LtJaFyUmJg7msCe4YGwSiZHBvLLh8JCPIYQQLrP5BYjNhszzLAvhjAlAa/0uMAVYArwCTNVan7EPQCmV2Nd3oJQKBeYDe84u3FML9PfjpmnprNhbzeG6Nmf9jBBCnL2qHVD6MUy9zWVTP5+Mo78cDNQDTUC+UmqOA99JAT5SSm0DPsX0Abw9tDAds3BGJv5K8dK6Emf+jBBCnJ0NT0JAKEy5zdIwAs60g1Lq18CNwE6gr4dVY5qCTklrvQ2YfLYBDsaI6BAuLRjBK5+W8e2LxhAefMZ/nhBCuFZbven8nXiDZZ2/fRwpIRcAeVprj5hr4Y5ZWby9rZLXPzvCopmZVocjhBAn2vwC9LTDjG9YHYlDTUAHMWP4PcKUjFgmpcfw1OqDMiRUCOFeujtg3eMw6nxIHm91NA4lgDZgi1LqL0qpR/s2Zwc2VEop7jk/h8P1bbyzvdLqcIQQ4ritf4OWo3De/VZHAjjWBLTMvnmM+eOSyU2K4PGVB7i6cCRKKatDEkL4ut4e+PgRGDkFsh0ZR+N8Z0wAWusXXBHIcPLzU9w9N4f/XrqVD3dXc1F+stUhCSF83c43zNTPF/8C3OSi1JG5gA4ppQ5+fnNFcGfjmkkjSY8L5Q8f7MNmc8oUREII4ZjeHlj1K0jKh7wrrI6mnyN9AEXANPs2G3gUWOzMoIZDoL8f375wDDsrmmTheCGEtba9AnXFcMEDlt749XmO3AlcN2A7orV+GHCfFHYaCyankpMYzu/f30ev1AKEEFbo6YSVv4aRk2GsexWdjjQBTRmwFSml7saxzmPL+fsp7p+fR3F1C298dsTqcIQQvmjTC9B4GOb9j9u0/fdxpCD/3YDnPcAh4AbnhDP8LisYwcS0aH773l4unzCCsCCPyF1CCG/Q3gAr/xeyZkPOhVZH8wWnrAEopf7T/vTHWusL7Nt8rfVdWuu9LorvrPn5KX58ZT5VTR38ZZXb910LIbzJqt+YJHDJ/3O7q384fRPQHfZHt73py1HTsuK4YmIKf1l9gMrGdqvDEUL4gtpi2PAXmLIIUiZaHc1JnS4B7FZK7QfylFLbBmzb7TN8epQfXDoWm4Zf/dNpM1ILIYShNfzrATPj57wfWx3NKZ2yQVxrfbNSagTwHnC160JyjvS4MO6em8OjH+7ny1PTOW90gtUhCSG81Z63Yd+7MP/nEOHQCrqWOO0oIK11lda6UGtd+vnNVQEOp3vPzyErPoz/eXM7Hd29VocjhPBGnc2w/HuQPAFm3mN1NKflPnckuEBIoD+//NIESuraeOyjYqvDEUJ4oxW/hOZKuOph8HfviZR9KgEAzMpN4EuTU3li1QH2VDVZHY4QwpuUbzIdv9PuhLQiq6M5I0duBPuyI+95kv+5YhxRIYHcv2QrXT2yZoAQYhh0t8Mb34DIkXDhg1ZH4xBHagA/dPA9jxEfEcz/XjuBXZVN/GnFfqvDEUJ4gw9+BnX7YcFjEBJtdTQOOeUoIKXUZcDlQOrnFoCJwtwR7NEuHj+C66ak8djKA1w4LpnC9BirQxJCeKpDq2H94zD9G2a1Lw9xuhpABbAR6AA2DdiWAZc4PzTn+8nV+SRHBvOdJVto6fT4nCaEsEJbPbx5L8TnwkU/tTqaQTllAtBab7UvBpOrtX7B/nwZUKy1bnBZhE4UFRLI72+cREldKw+8sR2tZcZQIcQgaG0K/+YquPYpCAqzOqJBcaQP4H2lVJRSKg7YDDyllPqDk+NymZmj4rl//hje2lLBkk/LrA5HCOFJ1j4G+/5pVvlKnWJ1NIPmSAKI1lo3AdcCL2qtZwDuN63dWbj3/Fxmj07gJ8t2srtShoYKIRxQ9il88BMYeyXM+IbV0QyJIwkgQCmVgpkC+m0nx2MJPz/FH26cRHRoIHcv3sSxti6rQxJCuLPmo7D0NogaCdc85pYzfTrCkQTwEGY+oANa60+VUqMArxs7mRARzOO3TKHyWAff/Ntn9PTK/QFCiJPo6YQlt5hpnm/8K4R67ghCR5aEXKq1nqi1vsf++qDW+jrnh+Z6UzPj+MWCAv5dXMsvl++2OhwhhLvRGt6+H8o3wII/u+00z45y5E7gNKXUG0qpavv2mlIqzRXBWeGGael8dVY2z31cwpJPD1sdjhDCnax/ArYshrnfh/Ffsjqas+ZIE9BzmOGfI+3bP+zvea0fXT6W2aMTeOCNHazaV2N1OEIId7D7H/DuD02n79wfWB3NsHAkASRqrZ/TWvfYt+eBRCfHZakAfz8eWziF3KQI7lm8ie3ljVaHJISwUulaeNU+wdu1T4Gfd8yj6ci/ok4pdYtSyt++3QLUOTswq0WFBPLCV6cTGxbEHc9voLSu1eqQhBBWqN4NL98IMRlw8xKPu9nrdBxJAF/FDAGtAiqB6zm+XrBXS44K4cU7p9Nj09z27AZqmjutDkkI4UqN5bD4OggIgVteg/B4qyMaVo6MAirVWl+ttU7UWidprRdorX2mdzQnMYJnbpvG0aZOFj69jroWSQJC+ISmCnj+SuhsgYWvQmym1RENO0dGAb2glIoZ8DpWKfWsc8NyL1MzY3nm9iJK69pY+PR6GlrlRjEhvFpzFbxwFbTWwqLXPX6456k40gQ0UWt9rO+FfSK4yWf6klIqXSn1kVJql1Jqp1LqP88mUKudm5PA07cVcbC2lVueWU9jW7fVIQkhnKGlBl64Gpoq4ZZXPWJlr6FyJAH4KaVi+17YJ4U75ToCA/QA/6W1zgdmAv+hlMofWpjuYfboRP6yaCr7j7aw6FmpCQjhdZqPwotXw7HDsPDvkDHT6oicypEE8DtgrVLq50qpnwOfAP93pi9prSu11pvtz5uB3UDq2QTrDi7IS+LxW6awp6qZG59cy9GmDqtDEkIMh4ZSePYS8/iVJZB1ntUROZ0jncAvYmYCPWrfrtVavzSYH1FKZWGajdYPPkT3c+G4ZJ6/fRrlDe18+Ym1lNW3WR2SEOJs1OyFZy+F9nq49U0YNdfqiFxCOXsRFKVUBLAK+KXW+vWTfH4XcBdARkbG1NLSUqfGM5y2lB3j9uc2EBzgx+I7ZzA6OdLqkIQQg1WxBRZfC8ofFr0BIwqsjmhQlFKbtNZD6qhw6u1sSqlA4DXgrycr/AG01k9qrYu01kWJiZ51g/Gk9BiW3HUOWsP1T6xl3UGvvz9OCO+y/wN4/goIDIevvutxhf/ZcloCUEop4Blgt9b69876HavljYjktXvOJTEymEXPrOf1zeVWhySEcMSnT8PfboC4bLjzPYjPsToil3NmDWAWsAiYp5TaYt8ud+LvWSY9LozX7j6XaVlx3P/3rfz+/X2yvrAQ7srWC+89AO/8F+ReBHe8axZ28UGODOccEq31vwHPXCZnCKLDAnn+jun86I3tPPrhfkrrWvn1dRMJCfS3OjQhRJ/OZnjjbtjzNky/Cy79Ffj57v+jTksAvigowI/fXD+R7IRwfvPeXoqrW3jilqmkx3nP5FFCeKzaYliyEGr3mYJ/5j1WR2Q575jT1I0opfiPC3J55rYiDte3cfWf/s2a/bKmgBCW2rMcnroAWqrNSB8p/AFJAE5z4bhkln3zPBIjg7nt2Q08vvKA9AsI4Wq2XljxS3jlZogbBd9YBaPOtzoqtyEJwImyE8J5495ZXDYhhV+/u4e7Xtok00cI4SrNR+Gv18Pq/4NJC80wz5gMq6NyK5IAnCw8OIA/3TyZH1+Zz8q91Vz2yBq5X0AIZyv+AJ6YBaWfwJUPwzWPQWCo1VG5HUkALqCU4s7zsnnj3lmEBvlz81Pr+P2/9tLTa7M6NCG8S08X/Ot/zCIu4Ylw10oougOUzwxIHBRJAC5UkBrN2/edx3VT0nh0RTE3PrlO5hESYrjUFsMz8+GTP8K0r8HXV0DSOKujcmuSAFwsPDiA3365kEdumsTeqmYufXg1i9eVSgexEENl64W1j5kmn4YSuHExXPE7afJxgCQAi1wzKZX3vjOHyRmx/M+bO1j0zAbKG6Q2IMSg1B2A5y6H934Eoy6A/1gP466yOiqPIQnAQqkxobx053R++aUCPjvcwKUPr+HlDYelNiDEmdhssO5xeHwW1OyGBU/AzS9D5AirI/MokgAsppRi4YxM3v32HCamRfPD17ez8On1HKhpsTo0IdzT0Z3w3KXw7g8gezbcuw4m3SwdvUMgCcBNpMeFsfjOGfxiQQHbjzRy2cNr+P37++jo7rU6NCHcQ1crvP8g/GUO1O6HBY/DV/7usxO5DQenLwgzGEVFRXrjxo1Wh2G56uYOfvnObt7aUkFWfBg/X1DA7NGetVaCEMNq7z9h+XehsQwmL4L5D0FYnNVRuQW3XRBGDE1SZAiP3DSZxXfOQCnFomc2cN/Ln1HZ2G51aEK4VkMJvLIQXr4JgiLM1M3X/EkK/2EiNQA319Hdy+MrD/D4qgP4Kbh7bg7fmJNDaJDvTmErfEBnM6z5nRne6RcAc74L53wTAoKsjsztnE0NQBKAhyirb+NX7+7hnW2VpESH8IPLxnJ14UiUdHwJb2LrhS1/gw8fgtZqKLwZLnxQ2vlPQxKAD9lwqJ6H3t7JjiNNTM6I4cdX5jMlI9bqsIQ4eyX/hnd/CFXbIG26mbM/barVUbk9SQA+xmbTvLq5nN+8t5ea5k4uzk/mvy/JY0xypNWhCTF4ldvMFX/x+xCVBvN/BgXXybBOB0kC8FGtnT088+9DPLn6IG1dPXxpchrfmT+atFhZgUx4gLoD8NEvYcdrEBID533HLNMYJH+/gyEJwMfVt3bx+MpiXlhbChq+MiODb87LJSEi2OrQhPiipkpY9Wv47CXwDzKrc537LQiNsToyjyQJQABQcaydRz/cz983lhEc4M+iczL5+uxRJEZKIhBuoPEIfPwIbHoetA2m3m5G90QmWx2ZR5MEIE5woKaFP364n2VbKwgK8OMr0zP5xtxRJEeFWB2a8EXHyuDffzBX/NoGhTfB7P+GuGyrI/MKkgDESR2qbeWxj4p547Mj+Pspbp6Wzt3n55ASLdPkChdoKIE1vzfDOgEmLzTt/LFZVkbldSQBiNM6XNfGn1cW8+qmcpSCL01O5a45o8hNklFDwgmqtptFWba/Cn7+MOU2mPWfEJNudWReSRKAcEh5Qxt/WXWQpZvK6Oi2ceHYJL4+ZxQzsuPkhjJxdrSGgx/Bx4+ax8BwmHobnHuf3MTlZJIAxKDUt3bx4toSXlxbSn1rF4Vp0dw1J4dLxicT4C/TQ4lB6O2GHa+bK/6j2yEiGWbcbdbhDZUbFF1BEoAYkvauXl7bXM7Taw5SUtdGelwot87M4oaTdkMXAAAWkElEQVSidKLDAq0OT7iz1jrY/AJ8+gw0lUNCnrnan3gDBMioM1eSBCDOSq9N8/6uKp5ec4iNpQ2EBvqzYHIqt5+bRd4I6ScQA1RsgQ1Pmvb93k7Imm0maRt9MfhJ7dEKkgDEsNlxpJEX15bw1pYKOntszBwVx+3nZnHROGke8lm93bDrLVPwl62HwDAzlHP6XZA0zurofJ4kADHsGlq7eOXTMhavK+XIsXZGRodw47QMbpiWJsNIfUVDqRm7v/klaKmC2GyY/nWYtFDu2nUjkgCE0/TaNB/sPsridaWs2V+Ln4J5Y5O4aVoG5+clSq3A2/R2m9W3Nj0PB1aYCdlyL4JpX4Pc+dLM44YkAQiXOFzXxpKNh/n7xnJqmjsZERXCDdPSuXFaOqkxUivwaPUHYfOL8NlfzTz8Ualm6cXJt8j4fTcnCUC4VHevjRV7qnl5w2FW7asB4LzcBK6bksYl40fIamWeorMZdi2DrS9DyRpQ/jDmEjNHT+5F5iYu4fYkAQjLlDe0sXRjOa9tLqe8oZ2I4AAunzCC66akMS0rDj8/ucHMrdhsULIatrwMu5dBdxvEjYLCr5ipGuSmLY/jlglAKfUscCVQrbUucOQ7kgA8l82m2VBSz2ubylm+vZLWrl7S40K5dnIa105JJTM+3OoQfVvtfnOlv3WJGbcfHAUF15qCP326LL7iwdw1AcwBWoAXJQH4lrauHt7bWcWrm8r55EAdWsOk9BiuLhzJlRNTSJJZSV2jsdzcpbvjVajcCsoPcuaZdXbHXgGB0m/jDdwyAQAopbKAtyUB+K6KY+0s21rBW1sq2F3ZhJ+CmaPiubpwJJcVpMgdx8OttQ52vQHbX4PDn5j3Rk6GguvNMotRKdbGJ4adRycApdRdwF0AGRkZU0tLS50Wj7BWcXUzy7ZUsGxrBSV1bQT6K+aOSeSqwpHMG5tEZIgkgyFpb4C975qlFQ+sAN1rpmaYYC/043OsjlA4kUcngIGkBuAbtNZsP9LIsi0VvL2tkqqmDoL8/ZgzJoHLClK4KD+Z6FBJBqfVWgd73zGjeA6uBFs3RKebAn/C9ZBcIO36PuJsEkDAcAcjxJkopZiYFsPEtBh+dPk4Nh9uYPn2Kt7dUckHu6sJ9FfMyk3g8oIU5ucnExseZHXI7qGlGva8baZlOLTGXOnHZJo1dfMXQOoUKfTFoEgNQLgNrTVbyxv55/ZKlu+opKy+HX8/xYzsOObnJ3PRuGTS48KsDtO16g6YO3P3LofDa82SinE5MH4BjLsaUgql0PdxbtkEpJR6GTgfSACOAj/RWj9zuu9IAhB9tNbsrGhi+fZK/rXrKMXVLQCMHRHJxfnJzM8fQUFqlPctZGOzQcVm2POOKfRr9pj3kwsg73JT8CflS6Ev+rllAhgKSQDiVA7VtvLBrqO8v+soG0vrsWlIiQ7honHJXJSfzIzsOEICPfTO1a42cyfu3uWmM7elytyVm3muGa6Zd5msoytOSRKA8Cl1LZ2s2FPN+7uOsmZ/Le3dvYQG+jMrN57z85K4YGyS+89N1FAC+/4F+/9lCv+eDgiKMFMw5F0Oo+dDWJzVUQoPIAlA+KyO7l7WHqzjoz3VrNhTTXlDOwB5yZFcMDaJC/ISmZoZa/2spT1dpg1/v73Qr91n3o/LMYupjLkYMmfJalpi0CQBCIHpNzhQ08pHe6r5aG81Gw7V02PTRIYEMCsngdljEpgzOtF1Hcl1B8y4/AMfwaHV0NUM/kGQdZ4p9EdfLGP0xVmTBCDESTR3dPNxcS0r99awel8NFY0dAGQnhDNndAKzRydyTk484cHDNBq6o9EU9AdWmK2hxLwfk2mmYBh9MWTPgeCI4fk9IZAEIMQZ9dUOVu+rYc3+GtYdrKe9u5dAf8WUjFhm5SZwTk48hWkxBAU42FzU3QHln5o2/IOrzHPda9rys2ZD7oWm4I8bJaN2hNNIAhBikDp7etlU0sCq/TWs2VfLrsomAEID/SnKiuWcnHjOGRXPhNTo4/0Hvd1wZLO5yi9ZDWUbTOctyozH7yvw06ZDgNy8JlxDEoAQZ6mhtYv1h+pYe6COtQfr2He0BX96mRZcxnVxB5mpdjGyaQv+PW3mC8kF5io/e44Zrilr5AqLyFQQQpyl2PAgLs1P5tL4GkjcTmfxKvwOf0JgTws0wH5bKotts9gaMBEyZzFhzCjOyYlnTFKkLHojPJYkAOG7bL1wdAccXmeadUo/NjNrAsFxOVB4vbnKz5pNpC2aqIO1BNhrCK/v3QVAfHgQM0fFMzMnnmlZsZIQhEeRJiDhOzpb4MhGOLzejMkv32iGZgLEZEDWHMg2BT7Rqac9VHlDm2kusieESvsIo8iQAKZkxDItK5apmXFMSo+RNZKFU0kfgBAn01Rhru7L1pvHqu1mlA4KksdD+gzIOAcyZpgEMERaa8rq29lYWs+nJQ1sKq1n31Ezd1GAn2J8ajRFmceTQmKk3Owlho8kACFsNqjZbQr6w+ugbB0cO2w+CwiFtCLImAnpMyF9GoREOzWcY21dbD7cwMYSs20tP0Znjw2AzPgwijLjKMqKpSgzlpzECGk2EkMmCUD4nvYGMyTzyCYzHLN8g7kRCyAi2X51P9NsIyaCv7ULzHT12NhR0cjGkno2ljSwqbSButYuAGLCApmaEcuUzFgmpccwIS2aKFkdTThIEoDwbj2dULXDFPZHNprHuuLjnyeOPX51nzEDYrPd/sYrrTWHalvZWNpgkkJpAwdrWgETek5iBIVpMUzKiGFSWgxjUyIJtHo+I+GWJAEI76E11B80HbR9BX7Vdug1V8tEJENqkVn9Kq3ILHju5OYcV2ls62Zr+TG2lB1ja5l57KslBAX4UTAyisL0GCbZt4y4MO9bD0EMmiQA4blaauwFfd/V/WboOGY+CwwzBXzqVLOlFUFUqttf3Q8XrTXlDe0nJITtRxr7+xJiwwL7E0JhegyFaTHEyfKZPkduBBOeoakSKrdC5RbzWLEFmivMZ8rPrHSVf7X9Cn+qadrx990/UaUU6XFhpMeFcVXhSAC6e23srWo2NYXDx9hafoxV+2rou45LjQll/MgoJqRGU5AWTcHIaBl1JE5JagBi+GkNTUeOF/J9hX7LUfsOChJGm/lzUibByEnmUWbJHJLmjm62lzey/YjZdlY0cai2tf/zEVEhFKRGUZAabRJDajTJUSEWRiyGk9QAhHW0NsMtP39l31ZrPld+kJBnJknrK/BHFEBwpLVxe5HIkEDOzU3g3NyE/veaOrrZVdHEjiON7LAnhg/3VPfXFBIjgymw1xTG2xNDSnSI9Cn4GEkAwnHd7VC920yfULXj+GOnffilXwAkjoMxl9qv6gvNpGlBLlqARfSLCgk0U1SMiu9/r7Wzh12VTf0JYeeRJlbtq8FmTwrRoYGMS4lkXEoU+SlRjEuJYnRyBMEBciezt5IEIE6u+Sgc3W5G4PQV9rX77XfSAoHh5m7aCdeZQj5lknkdKE0L7io8OIBpWXFMyzq+1nB7Vy+7q5rYeaSRXZXN7K5s4uUNh+noNh3NAX6KnMQI8kdG9SeHcSlRJERIv4I3kATg67rbzfq01bvh6E77Vf12aK05vk90uinkx10FIyaY57HZ4Cfj0j1daJA/UzJimZIR2/9er01TUtfK7somdlc2sauiibUH6njjsyP9+yRFBvcng7EjIhmTHElOUrjUFjyMJABf0dtjxtdX7zKFffUus9UfBG2u9vAPMiNvRl9i2umTC8xVfVjc6Y8tvIq//ao/JzGCKyeO7H+/vrWLPZVN7LJvuyub+eTAQbp7df/3shPCyUs2CSFvRARjkiPJjA/HX6a6cEuSALxNX6dsfyG/22y1e4/fTKX8zDKFSeOg4DrzmJRv3rN4ygThvuLCg77Q2dzVY6OkrpW9Vc3sO9rMnqpmdlQ0snxHZX+Hc3CAH7lJEeSNiDTJwf4onc7WkwTgqbQ2wypr9nyxsO9qOb5fVJop4HMuMIV80jhIzIPAUOtiF14jKMCPMfYr/oHaunoorm7pTwx7j7bwcXEtr28+3owUGRzAGHvzUV5yRP/z+PAgSQwuIvcBuDubDY6Vmnb6mr3mSr5mn3nsm/wMIDTONNf0Xc0n5UPSWK+ZJkF4h2NtXew72sLeo83sq2pm79Fm9lY109je3b9PdGgguUkR5CZGkJsUQU5SOLmJkaTGhkpT0knIVBDeoKcL6g/YC3l7YV+zF+r22xcetwtPNOPqE8fYH/NMwR+e6DNTJAjvorWmprmTPVXNFFe3UFzTQnF1CwdrWqht6erfLzjAj1GJEeQkhpsEYd+y4sMJCfTdzme5EcyTtNWbmSxr95vCve9qvv7Q8SGWYBYoSciDUXMhYYwp6BPGSIes8DpKKZKiQkiKCmHOmMQTPmto7eKAPSH0PW4tP8Y724/3MfgpSI8LO15jSIwgx54cokOlT+t0JAE4Q0+nGV1Tu98U9v0FfjG01x/fzy8A4nJMs03+AjMCJ3EMxI+Wm6eEAGLDgygKj6Mo68QLn47uXg7WtPbXFg7UtHCguoU1xbV02SfLA3PHc05iONkJfVsE2QlmfiUZsioJYOhsNjOR2cDCve95Y9nxoZUAESPM3Df5V5vCPWE0xOdCTKZPT3YmxFCFBPqTPzKK/JFRJ7zfa9OU1bf11xaKq1s4WNvKv3Ye7Z9aG0ytIS02jKyEcEbZk0Pf85ExvtPXIKXP6dhs0FIFdQfMFf3nt+624/sGhkN8jpmyuPBmU8An5JpHmfdGCJfw91Nk2QvzC8cln/BZY1s3h+paOVTbwqHaNg7VmuebSxto6ezp3y/I34+M+DCy7Qkhy54gRiWEkxgZ7FUjlCQB9F/Jn6yQPwQ97cf39QuE2CxT0GfPMYV7fK65oo9MkU5YIdxYdFggk8LM+gkDaa2paenkUE0rJXWtHKxt7X++al/NCU1K4UH+/QkhOyGcrPhwMuPDyIgL88jk4BsJwNYLjeUnv4qvPwS9ncf39Q+GuGxzU1TOPPvzHPM6Og38pN1QCG+ilCIpMoSkyBBmDJg8D0yTUsWxdkrqWjlU28pBe2LYfqSR5dsr+yfSAwgN9CcjLoyM+DAy48JMYogPJzMujNTYULdc0tOpCUApdSnwCOAPPK21/pXTfqyjERpK7FvpgOcl5s5Y2/FxxgSEmAI9PhdGzz9ewMeNgqiRUsgLIQDTpNS3KM/s0SeOUOrqsVHe0EZpfRtl9W2U1vVtrazZX9M/oR6YPoeRMaH22oKpNWTak0VGXBiRIdaMVnJaAlBK+QOPAfOBcuBTpdQyrfWuIR2wt9t0rp60gC+F9oYT9w+NNc01KRPNJGZ9BXzcKNNcIxOZCSHOQpD9voRRiV9cyEhrTXVzJ6V1bRyub+NwXSul9iTx3s4q6gd0SIOZZiPDXmswiSHc1CbiwkiKDMbPSZ3SzqwBTAeKtdYHAZRSrwDXAKdOALYeKN8EDYdOLNwbSkwTzsCRNX6BEJtpRtKkTjWFff+WKXfACiEso5QiOSqE5KgQpmd/8d6d5o5uSuvsNQd7Yjhc38qm0gb+sbXihKaloAA/0mJCSYsLIz021NRIYsNIjwslI+7shos7MwGkAmUDXpcDM077jart8PS846/Dk0yBnj4TJmaZgr2vkI9MkaYaIYRHigwJpMC+POfndfXYqDjW3t+0VFbfRllDG2X17WwrP8axtu6THHFoLO8EVkrdBdxlf9mpfta04/inTUCxBVG5hQSg1uog3ICch+PkXBwn5+K4vKF+0ZkJ4AiQPuB1mv29E2itnwSeBFBKbRzqnBbeRs6FIefhODkXx8m5OE4pNeQJ1JzZE/opMFopla2UCgJuApY58feEEEIMgtNqAFrrHqXUN4H3MMNAn9Va73TW7wkhhBgcp/YBaK2XA8sH8ZUnnRWLB5JzYch5OE7OxXFyLo4b8rlwq/UAhBBCuI7cDSWEED7K5QlAKXWpUmqvUqpYKfWDk3werJRaYv98vVIqy9UxuooD5+J+pdQupdQ2pdSHSqlMK+J0hTOdiwH7XaeU0koprx0B4si5UErdYP/b2KmU+purY3QVB/4fyVBKfaSU+sz+/8nlVsTpCkqpZ5VS1UqpHaf4XCmlHrWfq21KqSlnPKjW2mUbpjP4ADAKCAK2Avmf2+de4An785uAJa6M0c3OxQVAmP35Pb58Luz7RQKrgXVAkdVxW/h3MRr4DIi1v06yOm4Lz8WTwD325/lAidVxO/F8zAGmADtO8fnlwD8BBcwE1p/pmK6uAfRPD6G17gL6pocY6BrgBfvzV4ELlafNseqYM54LrfVHWuu+RQfWYe6l8EaO/F0A/Bz4NdBxks+8hSPn4uvAY1rrBgCtdbWLY3QVR86FBvpWhYkGKlwYn0tprVcD9afZ5RrgRW2sA2KUUimnO6arE8DJpodIPdU+WuseoBGIx/s4ci4GuhOT3b3RGc+FvTqbrrV+x5WBWcCRv4sxwBil1MdKqXX2WXe9kSPn4qfALUqpcsyIw/tcE5pbGmyZYv1UEOLMlFK3AEXAXKtjsYJSyg/4PXC7xaG4iwBMM9D5mFrhaqXUBK31MUujssbNwPNa698ppc4BXlJKFWg9cOZIcSqurgE4Mj1E/z5KqQBMta7OJdG5lkNTZSilLgIeAK7WWnd+/nMvcaZzEQkUACuVUiWY9s1lXtoR7MjfRTmwTGvdrbU+BOzDJARv48i5uBP4O4DWei0QgpknyBc5VKYM5OoE4Mj0EMuA2+zPrwdWaHsPh5c547lQSk0G/oIp/L21nRfOcC601o1a6wStdZbWOgvTH3K11nrIc6C4MUf+H3kTc/WPUioB0yR00JVBuogj5+IwcCGAUmocJgHUuDRK97EMuNU+Gmgm0Ki1rjzdF1zaBKRPMT2EUuohYKPWehnwDKYaV4zp8LjJlTG6ioPn4jdABLDU3g9+WGt9tWVBO4mD58InOHgu3gMuVkrtAnqB72qtva6W7OC5+C/gKaXUdzAdwrd76QUjSqmXMYk/wd7n8RMgEEBr/QSmD+RyzBTKbcAdZzyml54rIYQQZyB3AgshhI+SBCCEED5KEoAQQvgoSQBCCOGjJAEIIYSPkgQghBA+ShKAEEL4KEkAQgyglPqSUuqPA16HKqVWKaX8T7F/kFJqtX3aEiE8iiQAIU40Bdg84PVXgde11r0n29k+TfGHwI0uiE2IYSUJQAhAKTVGKfU+8G3gJ0qpb9s/Wgi8Zd/nbqXUFvt2SCn1kX2fN+37CeFRZCoI4fOUUsHAemARprA/F9gFZAH7tNYjPrd/ILAC+D+t9T/szUNVWutElwYuxFmSGoAQMB+z3GAF0KS1rsKsOhYJnGyO/Ucws9T+A8DePNSllIp0UbxCDAvpuBICCoHtwERgm1IqCWjGzKgYMnBHpdTtQCbwzc8dIxjvXqpSeCFJAEKYwn4i0ANsA36Gfc1dpZS/UipEa92hlJoK/Dcwe+CKU0qpeKBWa91tRfBCDJU0AQkBizEraj0I3INZh6JvKOi/gPPsz78JxAEf2TuCn7a/fwHg7WsVCy8kncBC2CmltgAXaa1rB7w3BfiO1nrRab73OvADrfU+F4QpxLCRGoAQ9I8Eih5Y+ANorTdjrvhPeSMY8KYU/sITSQ1ACCF8lNQAhBDCR0kCEEIIHyUJQAghfJQkACGE8FGSAIQQwkdJAhBCCB8lCUAIIXyUJAAhhPBR/x9g4ABYN51krwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# the cost of classifying a single-sample instance for different values of probabilites (phi_z)\n",
    "def cost_1(z):\n",
    "    return -np.log(sigmoid(z)) #when y =1, the cost function = -log(phi_z)\n",
    "def cost_0(z):\n",
    "    return -np.log(1-sigmoid(z)) #when y=0, the cost func = -log(1-phi_z)\n",
    "\n",
    "z = np.arange(-10,10,0.1)\n",
    "phi_z = sigmoid(z)\n",
    "\n",
    "c1 = [cost_1(x) for x in z]\n",
    "plt.plot(phi_z, c1, label = 'cost when y=1')\n",
    "\n",
    "c0 = [cost_0(x) for x in z]\n",
    "plt.plot(phi_z, c0, label = 'cost when y=0')\n",
    "\n",
    "plt.ylim(0, 5.1)\n",
    "plt.xlim(0, 1)\n",
    "plt.xlabel('$\\phi$(z)')\n",
    "plt.ylabel('cost func')\n",
    "plt.legend(loc = 'best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write logistic regression from scratch\n",
    "# convert adline to logistic regression\n",
    "\n",
    "class LogisticRegressionGD(object):\n",
    "    \"\"\"Logistic Regression Classifier using gradient descent\n",
    "    \n",
    "    Parameters\n",
    "    -------------\n",
    "    eta : learning rate\n",
    "    n_iter : epochs\n",
    "    randome_state : seed\n",
    "    \n",
    "    Attributes\n",
    "    ------------\n",
    "    w_ : weights after fitting\n",
    "    cost_ : logistic cost function value in each epoch\n",
    "    \n",
    "    \"\"\"\n",
    "    def __init__(self, eta=0.5, n_iter=100, random_state=1):\n",
    "        self.eta = eta\n",
    "        self.n_iter = n_iter\n",
    "        self.random_state = random_state\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \"\"\"Fit the traing data\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : training vectors\n",
    "        y : target\n",
    "        \"\"\"\n",
    "        rgen = np.random.RandomState(self.random_state)\n",
    "        self.w_ = rgen.normal(loc=0, scale=0.01, size=1+X.shape[1])\n",
    "        self.cost_ = []\n",
    "        \n",
    "        for i in range(self.n_iter):\n",
    "            net_input = self.net_input(X)\n",
    "            output = self.activation(net_input)\n",
    "            errors = (y - output)\n",
    "            self.w_[1:] += self.eta* X.T.dot(errors)\n",
    "            self.w_[0] += self.eta * errors.sum()\n",
    "            \n",
    "            #logistic cost\n",
    "            cost = (-y.dot(np.log(output)) - ((1-y).dot(np.log(1-output))))\n",
    "            self.cost_.append(cost)\n",
    "        return self\n",
    "    \n",
    "    def net_input(self,X):\n",
    "        return np.dot(X, self.w_[1:]) + self.w_[0]\n",
    "    def activation(self, z):\n",
    "        return 1/(1+ np.exp(-np.clip(z, -250, 250)))\n",
    "    def predict(self, X):\n",
    "        return np.where(self.activation(self.net_input(X)) >=0.5, 1, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# borrow X_train, y_train from the last notebook\n",
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "\n",
    "X = iris.data[:, [2,3]]\n",
    "y = iris.target\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state =1, stratify = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.4, 0.2],\n",
       "       [1.7, 0.2],\n",
       "       [5.3, 2.3],\n",
       "       [5.7, 2.1],\n",
       "       [1.2, 0.2],\n",
       "       [5.6, 2.4],\n",
       "       [6.6, 2.1],\n",
       "       [1.2, 0.2],\n",
       "       [5.8, 1.6],\n",
       "       [4.6, 1.3],\n",
       "       [3.3, 1. ],\n",
       "       [3.9, 1.4],\n",
       "       [1.3, 0.2],\n",
       "       [4.7, 1.2],\n",
       "       [4. , 1.3],\n",
       "       [5.1, 1.9],\n",
       "       [1.9, 0.2],\n",
       "       [3.5, 1. ],\n",
       "       [3.9, 1.2],\n",
       "       [1.6, 0.2],\n",
       "       [4.8, 1.8],\n",
       "       [6.9, 2.3],\n",
       "       [4.5, 1.5],\n",
       "       [4.5, 1.3],\n",
       "       [3.8, 1.1],\n",
       "       [5.6, 2.4],\n",
       "       [4.7, 1.4],\n",
       "       [5.6, 1.4],\n",
       "       [5.8, 2.2],\n",
       "       [5.1, 2.4],\n",
       "       [4.1, 1.3],\n",
       "       [1.5, 0.4],\n",
       "       [6.7, 2.2],\n",
       "       [5. , 1.9],\n",
       "       [4.8, 1.8],\n",
       "       [5.6, 2.1],\n",
       "       [4.2, 1.5],\n",
       "       [6.7, 2. ],\n",
       "       [5.5, 1.8],\n",
       "       [1.6, 0.2],\n",
       "       [4.9, 1.8],\n",
       "       [1.4, 0.2],\n",
       "       [5.1, 1.5],\n",
       "       [1.9, 0.4],\n",
       "       [1. , 0.2],\n",
       "       [4.9, 1.5],\n",
       "       [1.5, 0.2],\n",
       "       [1.1, 0.1],\n",
       "       [5.5, 1.8],\n",
       "       [4.7, 1.5],\n",
       "       [1.7, 0.3],\n",
       "       [3.5, 1. ],\n",
       "       [4.1, 1. ],\n",
       "       [1.5, 0.2],\n",
       "       [1.5, 0.2],\n",
       "       [1.5, 0.2],\n",
       "       [3.7, 1. ],\n",
       "       [6.1, 2.5],\n",
       "       [1.3, 0.3],\n",
       "       [4.9, 1.5],\n",
       "       [1.3, 0.2],\n",
       "       [4.5, 1.6],\n",
       "       [6. , 1.8],\n",
       "       [1.6, 0.2],\n",
       "       [1.4, 0.2],\n",
       "       [4.4, 1.2],\n",
       "       [5.1, 1.9],\n",
       "       [3.6, 1.3],\n",
       "       [4.1, 1.3],\n",
       "       [4.8, 1.8],\n",
       "       [6.1, 2.3],\n",
       "       [4.5, 1.5],\n",
       "       [4.5, 1.5],\n",
       "       [5.4, 2.1],\n",
       "       [5.6, 2.2],\n",
       "       [4.6, 1.4],\n",
       "       [1.5, 0.4],\n",
       "       [1.7, 0.5],\n",
       "       [6.1, 1.9],\n",
       "       [5.1, 1.6],\n",
       "       [6.4, 2. ],\n",
       "       [5. , 1.5],\n",
       "       [1.5, 0.2],\n",
       "       [4.2, 1.3],\n",
       "       [5.1, 2. ],\n",
       "       [6. , 2.5],\n",
       "       [1.5, 0.3],\n",
       "       [1.4, 0.2],\n",
       "       [5.6, 1.8],\n",
       "       [1.4, 0.3],\n",
       "       [5.9, 2.3],\n",
       "       [3.9, 1.1],\n",
       "       [4.2, 1.2],\n",
       "       [4.3, 1.3],\n",
       "       [1.5, 0.1],\n",
       "       [5.2, 2.3],\n",
       "       [4.4, 1.3],\n",
       "       [1.3, 0.2],\n",
       "       [1.4, 0.3],\n",
       "       [5. , 1.7],\n",
       "       [1.3, 0.3],\n",
       "       [4. , 1.2],\n",
       "       [1.6, 0.4],\n",
       "       [1.5, 0.2],\n",
       "       [1.3, 0.2]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "sc = StandardScaler()\n",
    "\n",
    "sc.fit(X_train)\n",
    "\n",
    "X_train_std = sc.transform(X_train)\n",
    "X_test_std = sc.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.colors import ListedColormap\n",
    "def plot_decision_region(X, y, classifier, test_idx = None, resolution = 0.02):\n",
    "    # setup marker generator and color map\n",
    "    colors = ('red', 'blue', 'lightgreen', 'gray', 'cyan')\n",
    "    markers = ('s', 'x', 'o', '^', 'v')\n",
    "    cmap = ListedColormap(colors[:len(np.unique(y))])\n",
    "    \n",
    "    # plot the decision surface\n",
    "    x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max() + 1\n",
    "    x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max() + 1\n",
    "    xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max, resolution),\n",
    "                           np.arange(x2_min, x2_max, resolution))\n",
    "    z = classifier.predict(np.array([xx1.ravel(), xx2.ravel()]).T)\n",
    "    z = z.reshape(xx1.shape)\n",
    "    \n",
    "    plt.contourf(xx1, xx2, z, alpha = 0.3, cmap = cmap)\n",
    "    plt.xlim(xx1.min(), xx1.max())\n",
    "    plt.ylim(xx2.min(), xx2.max())\n",
    "    \n",
    "    # plot class samples\n",
    "    for idx, cl in enumerate(np.unique(y)):\n",
    "        plt.scatter(x = X[ y == cl, 0],\n",
    "                    y = X[ y == cl, 1],\n",
    "                    alpha = 0.8,\n",
    "                    c = colors[idx],\n",
    "                    marker = markers[idx],\n",
    "                    label = cl,\n",
    "                    edgecolor = 'black')\n",
    "        \n",
    "    #highlight test samples\n",
    "    if test_idx:\n",
    "        #plot all samples\n",
    "        X_test, y_test = X[test_idx, :], y[test_idx]\n",
    "        \n",
    "        plt.scatter(X_test[:, 0], X_test[:, 1],\n",
    "                    c = '', edgecolor = 'black', alpha = 1.0,\n",
    "                    linewidth=1, marker ='o', s=100, label = 'test set')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.LogisticRegressionGD at 0x118348198>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_01_subset = X_train[(y_train == 0)|(y_train == 1)]\n",
    "y_train_01_subset = y_train[(y_train == 0)|(y_train == 1)]\n",
    "\n",
    "lrgd = LogisticRegressionGD(eta=0.05, n_iter=1000, random_state=1)\n",
    "lrgd.fit(X_train_01_subset, y_train_01_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEKCAYAAADuEgmxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3X94lPWZ7/H3zY+N+ANUBCWiRS0o09ZWS2e3VWzOcmrtj6DWdtXt7K6sezwLdrXrnh1se7UuZXtOM+fS3Xbb0HWrqEuXxgVR01q7bTTLSH8geFQ0VBZEK0aEEgpYMQK5zx/PM2GSTJJJMpNnfnxe1zVX5nnmm2fu4GXu3M/3nu/X3B0REZF8jIk6ABERKR9KGiIikjclDRERyZuShoiI5E1JQ0RE8qakISIieVPSEBGRvClpiIhI3pQ0REQkb+OiDqDQJk48xU89dUbUYYiIlJWtWzf+xt2nDDau4pLGqafO4I47NkQdhohIWZk/317OZ5xuT4mISN6UNEREJG9KGiIikreKm9PIZcyYQ0yevIPx49+KOpQBHTp0DHv2TKera3zUoYiI5FQVSWPy5B1Mm3YCEyfOwMyiDicnd2f//j3ADnbvPivqcEREcqqK21Pjx7/FxImTSzZhAJgZEydOLvlqSESqW1UkDaCkE0ZGOcQoItWtapKGiIiMnJLGKPrpTx/lAx84lwsvfCf/8A9fjzocEZEhU9IYJUeOHOFv//ZG/v3ff8QvftHG6tUr+dWv2qIOS0RkSKqie2ooLv/DD3Bg964+50+YMpWHHnty2NfduHE9Z5/9TmbMOBuAT33qGh555CHOOy827GuKiIy2yJKGmZ0B3AecCjhwp7t/o9eYOuAhYHt46gF3/2ox4zqwexfrT+m7Zlc8RyIZitdee5XTTz+j+7i2djobN/5yRNcUERltUVYah4G/cfenzOwEYKOZ/cTde9+zSbv7JyOIT0REeolsTsPdX3P3p8LnB4DNwOlRxVNs06adzquvvtJ93N6+g2nTKvbHFZEKVRIT4WY2A7gAyHW/5oNm9oyZ/cjM3jWqgRXQhRd+gG3b/ouXX97O22+/zQMPfJ+PfWx+1GGJiAxJ5BPhZnY8sBr4vLvv7/XyU8A73P0NM/s48CAwM8c1bgBuAJgy5cwiRzw848aNI5X6Fldd9VGOHDnCZz/758yeXbY5UESqVKRJw8zGEySM77n7A71fz04i7v6ImTWa2Snu/pte4+4E7gSYOXOOjySmE6ZMzTnpfcKUqSO5LACXXvpxLr304yO+johIVKLsnjLgLmCzu9/Rz5jTgNfd3c0sTnA7bc9A19254zCpxXuou2oy8fjQ4xpJW62ISKWLstK4CPgTYJOZPR2e+yJwJoC7fwf4NLDQzA4DB4Fr3H3ASuLcCb+mjsdpXRYjHtdnIERECimypOHuTwADrtDn7t8CvjWkC0+ezNKGGq5etJfUgjZq58X4/OdHEKiIiHQrie6pYmhq7KBuahuzNq3irQOHog5HRKQiRN49VUxLG2oAWGtd7HzpIKfNmBBxRCIi5a1iK41sU04+wvgxR9j56052/roz6nBERMpWVSQNgHPOPMx5Zx5kPIfY+dLBSGL43Of+nJkzp/LBD747kvcXERmpqkkaGeecefho1bEz95je/VkD92vl79prr2PVqkcLczERkQhUXdKAIHGc9nsdjH/7jT63q+65B7797aOJwj04vueekb/vRRddwkknnTzyC4mIRKQqkwbAiadN4JwzD0NXFzt/3cnvfhckiDfegNWrjyaOb387OH7jjcJVHCIi5aqiu6fycd6MTrb9ehxv7TnEgT3jufHGoONq9ergAXDVVXDjjWADfqpERKTyVW2lke2cMw93Vx2vv9LJjTf2fF0JQ0QkoKSR5bwZnfiRLr7+94c59HZX9/nsOQ4RkWqmpJHFHR5Z8zYtP+3iEx89wr/dd5irruo5xzES119/LZde+kG2bn2Bd71rOv/6r3cVJnARkVFS9XMa2czghJPHc+1njnDLjZ288HINV10OBw6M4/jjR36L6q67VhYmUBGRiChp9HLDdW/jHiSI86Z08NsDY7jumhOx8ePRP5eIVDvdnsqhu6I47jhOPG0Cs6d2wOFD7NxxmN/9LtLQREQiVTVJY5BtOAZ23HGcN6WD8V1vcWD3QXbuOFy4wLKMKEYRkVFQFUnjmEOH2LN//4gTxzlnHua8KUerjkJyd/bv38OhQ8cU9LoiIoVUFTfpp+/Zww5g929+M+jYvLz9a3YeOI7Xdo9n4sTCXBLg0KFj2LNneuEuKCJSYFWRNMZ3dXHW7t0FvebsFf/I3JbbYHawpWwyWdDLi4iUpKq4PVUUiQTpeUtI0gCb20ilog5IRKT4lDRGIpGgPhkjPW9Jd+JYvz7qoEREikdJoxDCqqP2pXW0LlPVISKVK7KkYWZnmNnjZtZmZs+b2c05xpiZfdPMtprZs2Z2YRSx5iWRoKmxo7vqWLEi6oBERAovykrjMPA37h4D/gC40cxivcZ8DJgZPm4Alo1uiMOQSJCc3Ux7SxupRdtVdYhIRYksabj7a+7+VPj8ALAZOL3XsMuB+zzwC+BEM5s2yqEOWX0yRnrhStIfulVVh4hUlJJouTWzGcAFwC97vXQ68ErW8Y7w3Gu9vv8GgkqEM6dMKVaYQxOPQzxOsr2ZVAuk2mPU1kIiEXVgIiLDF/lEuJkdD6wGPu/u+4dzDXe/093nuPucKZMmFTbAEapPxkjObiZJA+0tqjpEpLxFWmmY2XiChPE9d38gx5BXgTOyjqeH58pKfTKcqkkFVccKYsyaFRQjIiLlJMruKQPuAja7+x39DHsY+NOwi+oPgH3u/lo/Y0tepurgZ0FrrqoOESk3UVYaFwF/Amwys6fDc18EzgRw9+8AjwAfB7YCbwILIoizoOqTMerpoDn1BKkWWD8rpopDRMpGZEnD3Z8ABtwLz4NlaW8cnYhGV30yxvrFbbQug9bVp1L7nsmaJBeRkhf5RHg1W9pQQ3rhSpKTv0t7S5uWIBGRklcSLbdVLR6nPk531bFliybJRaR0qdIoEUsbakjObmbWplW0LlPVIVJovfdg00aZw6OkUULqkzGWNtRQN7VNiUOkgB59FB588GiicA+OH3002rjKkZJGCcpOHKkFSh4iI+EOBw9COn00cTz4YHB88KAqjqHSnEaJWtpQA2zjy4s7aV0G8XjvtRxFJB9mcMUVwfN0OngAzJ0bnLcBezilN1UaJW5pQw21E/aSWtBGavEeVR0iw5CdODKUMIZHSaMMZPbpqONxWpe1RR2OSNnJ3JLKlj3HIflT0igXiUSPqkNLkIjkJ3sOY+5cuP324Gv2HEeu7xnouJopaZSZpsYO6qa2wc/WkVqgqkNkMGYwYULPOYwrrgiOJ0zoe4tKnVYDU9IoQ0sbamhq7OiuOkRkYJdd1nMOI5M4Lrus5zh1Wg1O3VNlrKmxg6sXESSOCRNINp4VdUgiJat3RZFrElydVoNTpVHmmho7SC/fRi3tpBZtjzockbKnTquBKWlUiKbGDjh4kNSi7aRSUUcjUr7UaTUwJY0Kkl6+jfSHbqX2pXWqOqSijFY301A6rao1iShpVJpEokfVIVLuRrObqb9Oq6lT4YUXjo6r5o4qJY0KlV6+LUgci/dEHYrIsEXRzdS70wpg1izYtUsdVaDuqYqWXriSucuuJbXgdXVXSVmKqpsp+7pmcOWVwVd1VKnSqGzxeDDPoapDylgpdDOVQgylQkmjSqQXroRdr5NavEfJQ8pKKXQzlUIMpSLSpGFmd5vZLjN7rp/X68xsn5k9HT6+MtoxVox4nPTClaTfs6g7eYiMluF2P2V+Wbe29uxmam0tzLpRXV0DH2fHMJS1qypZ1JXGPcBlg4xJu/v7wsdXRyGmyhWPQyJxtOpIoYUPpehG0v1kBj/8IXR0QH19cFxfHxz/8IcjWzfq9tvha187mii6uoLj22/vG8NQ1q6qdJEmDXdfC3REGUNVCquOuj2raG9pU9UhRTPS7qcjR6CmBvbvh5tvDo5vvjk4rqkJjofzXl1d8Oab8OyzRxPH174WHL/5Zt+KI9+1q6qBecS1lZnNAH7g7u/O8VodsBrYAbQD/8vdnx/oenNmzvQNd9xR+EAr1fr1zF12LcyOkUxGHYxUouxf3hlD6Tw6cgRuugl27Dh6bvp0+OY3YezY4b9XdqLIOP98+NKXYEzU92AiMH++bXT3OYONK/V/mqeAd7j7e4F/Ah7MNcjMbjCzDWa2Yfe+faMaYNmLx0nPWwKb27QEiRTFSDuPxo4NEkS2XAljqO81ZkyQILJVa8IYipL+53H3/e7+Rvj8EWC8mZ2SY9yd7j7H3edMmTRp1OMse4lE9xIkbG5T4pCCGmnnUabSyHbTTT1vTQ3nvTKVRrbsOQ7JraSThpmdZhb8jWBmcYJ4dQO+WBKJo1VHCiUPAUa27tNIO4+yb01Nnw4PPBB83bEjOH/4cM/3WrMG1q7t+V5r1/Z9r+xbU+efD01NwdfsOQ7JLeqW25XAz4FzzWyHmV1vZn9pZn8ZDvk08JyZPQN8E7jGo56EqXRh4kjSoKpDRrzu00g7j8aOhWOO6TmH8c1vBsedndDc3DMZrF0bJJLLLw+uffnlcPLJ8OKLPd9rzBg49tiecxhf+lJwfOyxukU1kMgnwgtNE+EFtGIFc1tuo3ZejFmzgo5dqR69q4Qrruh7nO+8hHvPsb2PB3PkSM85jMOHg4TRO7aHHw5er68Plv548MEgkVxySf+T4dkJovdxNcl3IlxJQwbUnGpjxUsX037wJHVYVaGRdj4VU3+xucMTT/Q8VwrxlrqCdk+Z2VgzqzWzMzOPkYco5aA+GQt2BwznOvRhwOpSymsu9RfblVf2PVcK8VaKQZOGmf0V8DrwE+CH4eMHRY5LSk0iQXJ2c/BhwEXblTyqRCmvudRfbGvW9D1XCvFWinyWRr8ZONfd1bVU5eqTMerXr6S59XhSLfWsIEYiEXVUMpjhzicMNKcBwSRz9v1/9+CRa44gVwyQO658xnZ1wUMP9Zyv6G9OIxOvKo7CyCdpvALoE3MSiMepjwOpZlItkGqPUVuLkkeJevTRYAmNzC/MTCKYMGHwJTD663yCoBvpoYd6Xvev/zqYsP7GN4JEkWlr/e1v4eKLe45taAius3hxz7hefBHOPnvwsQ89FMxbnHxyz06pxx4LJswz+19k4q3GNaKKpd/bU2Z2i5ndArwItJrZFzLnwvNSxeqTMZKzm0nSQHuL5jpKUSF2vcu15tLllwe/2LOv+8ADsHNn8PmJ7LWcnnkG3nqr52cl1qwJtk594YWeca1dG6z7lD32wQePjl2zpufYE0+EPXuCBJJJJOPGBZVHRjWvEVUs/XZPmdltA3yfl+qKs+qeGn3NqTZSm+upWxgD1JpbSorV/ZTruhddBG1tfddy+uIXg9tG2WMvvvjoTnjZcV1+efDLP5+OqP7G6jbU8Iy4e8rdl7j7EqAt8zzr3OZCBivlrT4Zo25qG1vuWUfrMlUdpaRY3U+5rvupT+Vey2ns2L5jr7wyd1xjxuTfEdXfWCWM4sqn5fYLeZ6TKra0oYamxo7uDqv166OOSKB43U+5rvvAA7nXcjpypO/YNWtyx9XVlX9HVH9j1SlVXP1OhJvZx4CPA6ebWfYakxOBw7m/S6pdfTLG+sVttC6D1tWnUnfVZN2uishg3U/5/lXeu5spV+fSAw/A/ffD22/DhRcGFUZmTuOmm2D8ePjwh4Oxa9YEn+YGmD//aFxr18JzzwUbLA3WEdXf2HQ6iDczEZ4rfhmZgSqNdmAj8Fb4NfN4GPho8UOTcrW0oSbY5InHaV2mqiMqhdhxLtfaU7k6l668MrjmpEk913J673uDNaImTz469oor4Pjjg7HZcV1ySdBp1bsjKjM2uyPqkkuCNaKylwe54gqYOhW2bDka/1DXypLB9VtpuPszwDNm9j13PzSKMUkliMdZGocvh1XHli1avyoKl13W8y/tzC/XfCuMTPcV9KwITj45+Cs/03b70ENw0klBQspce8yYo5Pga9f2HDtuXDA2I5MgIPfYXB1RuT7Tce65R7u6eq+VpYqjMAbqntoE9Ht30N3PL1ZQI6HuqdLTnGpj/Z5zaN0Vo25hTImjjPTXfTWUzqWhdHCNtNurlNfKKnWFWHvqk0A98Gj4+Gz4+BHwSCGClOpQn4yxtKGGuqlttC5rizocGYL+uq+G0rk0lA6ukXZ7lfJaWZVioJbbl939ZeAj7p50903hYzFw6eiFKJViaUMNtRP2klqgxFEu+uu+Gkrn0lA6uEba7VXKa2VVinyWETEzu8jd14UHH6LEd/yT0tXU2MHViwgSx4QJJBvPijokyZJ93z/z6e10umeH0n/+Z/+dS9DzL/uhdHCNtNurUN1iMrB8ksb1wN1mNgkwYC/w50WNSipaU2MHrH+Uq++5jNSCgySXx6IOSei7ThUEk9JHjvTsZnruuaDLqXfnEvTtyhpo/aqRjM1lpN8v+cl7E6YwaeDuJb14oSbCy8vVi06mnVqYcZY2eIpQrr/SB/o8xSWX5F7ltr9fzENZaXeku/yN9Pur1Yh37jOzhLuv6G9xQncvyd/MShrlJ3t3QFUd0cnVedTfGlG61VN5CtE9dVz49YR+HiIFkdkdsHbCXlKLtuvDgBHJ1XnU3xpRShjVa6AP9/1z+LTB3d8apXikimUmybfc007rPbWaJB9luTqP1qzpmyAyH5xT4qhO+XRBPWdm68zs62b2iczcRiGY2d1mtsvMnuvndTOzb5rZVjN71swuLNR7S2lqauwIJsoPHiS1aHvU4USu993jgaYgu7oGPh7oOpkW2sycxu23B7emmpuDT3Rnzs2d23MfjZHEK+Vp0KTh7u8ErgU2AZ8gWFrk6QK9/z3AQNujfAyYGT5uAJYV6H2lxKWXb+tOHKnF1bnTcK51n/pbR+n2249ufgRHN0G6/fb8rvvQQ8GuednzFVdeGSzLce65+a1dNZR4pXwNmjTMbDpwETAXuAB4HmgqxJu7+1qgY4AhlwP3eeAXwIlmNq0Q7y2lL718G+nr7oZdr1dd1TGUXfe6uoId7559tueuec8+G5zPrjgGuu7ZZx9trYXg6+LFR7dZzZzLtRNeIXYJlPIwaMutmXUBTwL/290fKngAZjOAH7j7u3O89gPg6+7+RHjcAix29w39XU/dU5Vp7oJzYOqp1L5nctXsRz6UdZSyE0XG+ecfXXF2uNctVrxSegrRPZVxAXAf8Mdm9nMzu8/Mrh9xhAVkZjeY2QYz27B7X0l/jESGKbPUentLW9VUHUNZRymzFHm2XAljqNctVrxSvvKZ03gGuBdYDjwGfBj4SpHjyngVOCPreHp4rgd3v9Pd57j7nCmTCjZPL6UkHg/26cjMdVTBPMdQ1lHKVBrZsuc4hnvdYsUr5SufOY0NwM+BKwn2Br/E3d9R7MBCDwN/GnZR/QGwz91fG6X3lhKVXrgymOdY0FaxyaP3J7QH6lzKvjV1/vnQ1BR8zZ7jGM51ixWvlLd81p76mLvvLsabm9lKoA44xcx2ALcB4wHc/TsES7B/HNgKvAksKEYcUmbicdLxbbB+PXOXXUtqMSQbJkcdVUENZR2lMWOCXeyy5zAy260ee2zPW1TFWp9J6z5Vj7zXnioXmgivMmHiYOqpQOUlj6Gso9TV1TNB9D4e7nWLFa+UlkJOhIuUrnic9MKVpN+zKLhlVWG3q3r/wh3oF3DvBNFfwhjqdYeiWNeV0qGkIeUvHodE4uhcRwpWrIg6KJHK1O+chpl9aqBvdPcHCh+OyAjE46S3LOHLmz5D6+YYqfaYllsXKbCBJsLrB3jNASUNKT2JBEsBVixhbsttpFJKHCKFNNAqt+pUkvKVSJAmTBwLgNlKHiKFkNecRri6bdLMvpJ5FDswkRFLJIL1q+Ytgc1tpFJRByRS/vL5cN93gKuBvyLYI/wzwGh9uE9k5BKJHolDyUNk+PKpND7k7n8K7HX3JcAHgVnFDUukwMLEkaRBVYfICOSTNA6GX980s1rgEKDlyaX8JBLUJ2PdVceKFWhrWZEhyidp/MDMTgT+L/AU8BKwsphBiRRVIkFydjP8bB2ty9r0mQ6RIchn7amUu3cCq8P9LY4BtGe4lLX6ZIx6OmhOPUGqBVYQq5p9OkRGIp9K4+eZJ+7e6e77ss+JlLP6ZIzk7GbaW1RxiORjoE+EnwacDkwwswsIOqcAJgLHjkJsIqOiPhmDVDOpFkhtqq7dAUWGaqDbUx8FriPY+Ch72dj9wBeLGJPIqKtPxqhfsYTm9gtJtdSzflaMeDzqqERKz0CfCL8XuNfMrnL31aMYk0g0EgnqgfWL22hdBlu2xDKnRSSUz5zGOjO7y8x+BGBmsVLbI1ykkJY21FA3tY1Zm1bR3tKmtlyRLPkkjeXAj4Ha8HgL8PmiRSRSApY21HQnj9ZlShwiGfkkjVPc/X6gC8DdDwNHihqVSInIThypRduVPKTq5fM5jd+Z2WSC5dAxsz8A9hU1KpESsrShBtjGlxd30rrsIKBJcqle+VQatwAPA+eY2TrgPoLFC0WqSo+qI6UlSKQ6DZo03P0p4MPAh4D/CbzL3Z8txJub2WVm9oKZbTWzW3O8fp2Z7Tazp8PHXxTifUWGa2lDDel5S6jbs4rWZW1RhyMy6ga9PWVmxwCLgIsJblGlzew77j6ipUTMbCzwbeAjwA7gSTN72N17/5/Y5O6fG8l7SXH84S23sH9f3zuVEydN4rE77sjxHRUi3B3w6kV7SS1oo25h0JqrW1ZSDfKZ07gPOAD8U3j8x8C/EuyrMRJxYKu7vwhgZt8HLgf051uZ2L9vHxsmTepzfk6ORFKJmho7+PLi19hyz17aD55EPB6LOiSRostnTuPd7n69uz8ePv4H8K4CvPfpwCtZxzvCc71dZWbPmtkqMzujAO8rUjBLG2poauygdkJQdYhUunySxlNhxxQAZvb7wIbihdRDMzDD3c8HfgLcm2uQmd1gZhvMbMPuKvkrV0pLd+JYtJ3Uou1RhyNSNPkkjfcDPzOzl8zsJYIVbj9gZpvMbCQT4q8C2ZXD9PBcN3ffEy7LDvDdMJY+3P1Od5/j7nOm5LhdIjIamho7SF93N7W0q+qQipXPnMZlRXrvJ4GZZnYWQbK4hmC+pJuZTXP318LD+cDmIsUiUhjxOE3xDq5eRFBxzDiLZDLqoEQKZ9Ck4e4vF+ON3f2wmX2OYImSscDd7v68mX0V2ODuDwM3mdl84DDQQbDqrhTR6Z/5DBw61Of8W+6cdcopPc69vncvp+3dy/STTupxfmKOaq/aOq2aGoMNnla8dDGpRbUkG8+KOiSRgsin0igad38EeKTXua9kPf8C8IXRjquqHTrEq+PH9zl9Wmdn306pSZOYs28fG+66a9DLVmOnVWZ3wLkLTiK1aDt1152ltlwpe/nMaYjICKSXb6OWdrbcs06T5FL2lDRERkFTYwdNjR1w8KASh5Q1JQ2RUZRevq07caQW74k6HJEhU9IQGWXphStJX3c37HpdVYeUnUgnwmV09NcR9WZXF8eO6fl3w5GuLk7q7KSm19hO+k5av753L4eAOdf33MgxV0fUxHDSvLdcnVYVL5wNT8e3MXfBOaQW76H2PZO1rayUBSWNajBAR1Tv8wc7O3kn8Oo739njfK4uqTnXX593R1QlttUWQnrhSppbjyfVUk9q06kkGyZHHZLIgHR7SiRK8Tj1yRjphSuD21Wa55ASp6QhUgri8aOJY0GbkoeULCUNkVIRj5Nevk1Vh5Q0JQ2RUpNddSzeo+QhJUUT4WVgKOs25eqUerOri9M6O+mtE/qcfxsYD5y2dWvfsfPn9zjnwMm7dzOhVwcWOSbdq23tqRGLx0lvWQLA3JbbSKUma+FDKQlKGmVgSOs25eiUOq2zk9cA6zX0NGBnr3PvBf6NvrtsnQbstJ5X6HKnFnj17LMHjasa154asbAHNz1rJXOXXUsqFVPikMjp9pRIqYvHSc9bApvbSKVg/fqoA5JqpqQhUg4SCdLzllD70jpalwXJQyQKShoi5SKRCHYHDKuOFSuiDkiqkZKGSLlJJEjObqa9Jfg8h6oOGU2aCC8DudZt2rF3L+Ppu+7TW+6c3qt7qhOYluO6nQQT3L3PzetvrHvO871jy7WelNaeKqz6ZIz69SthyxbmttzGihUxrV0lo8I8xy+CcjZn5kzfUAUtnAOt+5TPTnqjfV0pnuZUG6nN9dTOiwEoeciwzJ9vG919zmDjdHtKpMzVJ2MkZzcza9Mq2ls01yHFpaQhUgHqkzGWNtR0z3UocUixKGmIVJBM1dHe0kZqgZKHFF6kScPMLjOzF8xsq5ndmuP1GjNrCl//pZnNGP0oRcpLfTJGevm27uShDwNKIUXWPWVmY4FvAx8BdgBPmtnD7t6WNex6YK+7v9PMrgEagKtHP9rSU6xuJHU5VY76ZIz1i9toXQatq0/V7oBSEJF1T5nZB4G/c/ePhsdfAHD3/5M15sfhmJ+b2TiCpZKm+ABBV0v3lEjeVqyguf1CUpvrqVsYy+w2K9JDvt1TUX5O43TglazjHcDv9zfG3Q+b2T5gMvCbUYlQpBIkEtRDd9WxZUuMWbNQ8pBhqYiJcDO7wcw2mNmG3Vo1VSSnpQ011E1tY9amVbQu01yHDE+USeNV4Iys4+nhuZxjwttTk4A+O9K4+53uPsfd50zRvXeRfi1tqOlOHkocMhxRJo0ngZlmdpaZ/R5wDfBwrzEPA38WPv808NhA8xkikp/sxJFatF3JQ/IWWdJw98PA54AfA5uB+939eTP7qplltoi7C5hsZluBW4A+bbkiMjxLG2pIL99G3QkbaV3WNvg3iKC1p0QEuHrRybQfPAlmx6ir0yR5NdLaUyKSt8w+HXV7VqnqkAEpaYhIIJFgaUMNtRP2klqgSXLJTUlDRHpoauygbmobW+5ZR2qBqg7pSUlDRPpY2lBDU2NHd9UhkqGkISL96k4ci7aTWrQ96nCkBChpiMiAmho7SF93N7W0K3GIkoaI5CEep6mxAw4eDKqOVNQBSVSUNEQkb+nl20gseijIAAAJmElEQVTOuJ/al9ap6qhSShoiMiT1yViPqkOqi5KGiAxLevm2IHEs7rOGqFQwJQ0RGbb0wpWw63VSC9pUdVQJJQ0RGb54nPTybao6qoiShogURHfVsXiPuqsqmJKGiBRGPE564UrS71kEm9tUdVQoJQ0RKZx4HBKJHlWHFj6sLEoaIlJ4YdVRe+BXwe6AqjoqhpKGiBRH+Cny7qpD8xwVQUlDRIorHic9b0kwz6ElSMqekoaIFF84z5G+7u4geShxlC0lDREZHfF4z6ojhZJHGVLSEJHRlUiQnreEJA2qOspQJEnDzE42s5+Y2X+FX0/qZ9wRM3s6fDw82nGKSJEkEtQnY91Vx4oVqDW3TERVadwKtLj7TKAlPM7loLu/L3zMH73wRGRUJBIkZzfDz9YFrbmqOkpeVEnjcuDe8Pm9wBURxSEiEcsstZ5ddUjpiippnOrur4XPdwKn9jPuGDPbYGa/MLN+E4uZ3RCO27B7376CBysioyCsOtpbgtZcJY/SNK5YFzaznwKn5XjpS9kH7u5m5v1c5h3u/qqZnQ08Zmab3H1b70HufidwJ8CcmTP7u5aIlLj6ZIz69Stpbj2eVEs9K4iRSEQdlWQrWtJw9//e32tm9rqZTXP318xsGrCrn2u8Gn590cxagQuAPklDRCpIPE59HEg1k2qBFcQAlDxKRFS3px4G/ix8/mfAQ70HmNlJZlYTPj8FuAhoG7UIRSRS9ckYydnNJNobaG/RXEepKFqlMYivA/eb2fXAy8AfAZjZHOAv3f0vgNnAP5tZF0Fy+7q7K2mIVJH6ZFBlZKqO9bNixOPRxlTtzL2ypgDmzJzpG+64I+owRKTAvry4k9ZdQRKpnae5jkKbP982uvucwcbpE+EiUhaWNtSQXr6tu8NKHwaMhpKGiJSV+mSMuqlt3ft0KHmMLiUNESk7SxtqSM9bQh2P07pMVcdoUtIQkfKUSLC0oaa76tD6VaNDSUNEylomcczatEpVxyhQ0hCRsre0oaZH1SHFo6QhIhVjaUMNtRP2kloQrF+lqqPwlDREpKI0NXaQXr6NuhM2quooAiUNEalI2VWHFI6ShohUrKbGjiBxaKn1glHSEJGK1tTYQXLG/fCzdao6CkBJQ0QqXmZ3wEzVIcOnpCEiVaOpsYNa2ru7q2TolDREpKpkuqs4eFCJYxiUNESkKmUnjtTiPVGHUzaUNESkaqWXbyP9oVth1+uqOvKkpCEi1S2ROFp1LN6j1txBKGmIiADphSup43HaW9pIpaKOpnRV3HavZnYAeCHqOIroFOA3UQdRJPrZypN+tvLU+2d7h7tPGeybxhUvnsi8kM8+t+XKzDZU6s+nn6086WcrT8P92XR7SkRE8qakISIieavEpHFn1AEUWSX/fPrZypN+tvI0rJ+t4ibCRUSkeCqx0hARkSKpqKRhZpeZ2QtmttXMbo06nkIxs7vNbJeZPRd1LIVmZmeY2eNm1mZmz5vZzVHHVChmdoyZrTezZ8KfbUnUMRWamY01s/9nZj+IOpZCMrOXzGyTmT1tZhuijqeQzOxEM1tlZr8ys81m9sEhfX+l3J4ys7HAFuAjwA7gSeBady/7BfTN7BLgDeA+d3931PEUkplNA6a5+1NmdgKwEbiiQv67GXCcu79hZuOBJ4Cb3f0XEYdWMGZ2CzAHmOjun4w6nkIxs5eAOe5ecZ/RMLN7gbS7f9fMfg841t1/m+/3V1KlEQe2uvuL7v428H3g8ohjKgh3Xwt0RB1HMbj7a+7+VPj8ALAZOD3aqArDA2+Eh+PDR2X8lQaY2XTgE8B3o45F8mNmk4BLgLsA3P3toSQMqKykcTrwStbxDirkl0+1MLMZwAXAL6ONpHDC2zdPA7uAn7h7xfxswD8CSaAr6kCKwIH/MLONZnZD1MEU0FnAbmB5eFvxu2Z23FAuUElJQ8qYmR0PrAY+7+77o46nUNz9iLu/D5gOxM2sIm4vmtkngV3uvjHqWIrkYne/EPgYcGN4i7gSjAMuBJa5+wXA74Ahzf9WUtJ4FTgj63h6eE5KXHi/fzXwPXd/IOp4iiG8BfA4cFnUsRTIRcD88N7/94E/NLOKWR/W3V8Nv+4C1hDc/q4EO4AdWRXvKoIkkrdKShpPAjPN7Kxwcuca4OGIY5JBhJPFdwGb3f2OqOMpJDObYmYnhs8nEDRp/CraqArD3b/g7tPdfQbB/2uPuXsi4rAKwsyOC5syCG/dXApUROeiu+8EXjGzc8NT84AhNZ1UzIKF7n7YzD4H/BgYC9zt7s9HHFZBmNlKoA44xcx2ALe5+13RRlUwFwF/AmwK7/0DfNHdH4kwpkKZBtwbdvaNAe5394pqTa1QpwJrgr9nGAf8m7s/Gm1IBfVXwPfCP65fBBYM5ZsrpuVWRESKr5JuT4mISJEpaYiISN6UNEREJG9KGiIikjclDRERyZuShkiezOw6M6vNY9w9ZvbpfM8XIK4vZj2fUYmrIUvpUNIQyd91wKBJIwJfHHyISGEoaUhVCv8i/5WZfS/cU2CVmR0bvvZ+M/vPcLG6H5vZtLBCmEPwoainzWyCmX3FzJ40s+fM7M7w0+35vn+f9wjPt5pZQ7gPxxYzmxueP9bM7g/3HVljZr80szlm9nVgQhjT98LLjzWzfwn38PiP8NPoIgWhpCHV7Fyg0d1nA/uBReE6WP8EfNrd3w/cDXzN3VcBG4DPuvv73P0g8C13/0C4x8kEIK/9JPp7j6wh49w9DnweuC08twjY6+4x4MvA+wHc/VbgYBjTZ8OxM4Fvu/u7gN8CVw39n0Ykt4pZRkRkGF5x93Xh8xXATcCjwLuBn4SFw1jgtX6+/7+ZWRI4FjgZeB5ozuN9zx3kPTKLNm4EZoTPLwa+AeDuz5nZswNcf7u7Z5Zkyb6GyIgpaUg1672GjgMGPO/uA26BaWbHAI0Eu7u9YmZ/BxyT5/sO9h6d4dcjDO//0c6s50cIqiCRgtDtKalmZ2btj/zHBNuxvgBMyZw3s/Fm9q5wzAHghPB5JkH8JtwLZChdUQO9R3/WAX8Ujo8B78l67VB4y0uk6JQ0pJq9QLDBzmbgJIKNad4mSAANZvYM8DTwoXD8PcB3wtV4O4F/IVgy+8cES/PnZZD36E8jQaJpA/6e4FbYvvC1O4FnsybCRYpGq9xKVQq3lv1BOIld8sLl1ce7+1tmdg7wU+DcMAGJjBrNaYiUh2OBx8PbUAYsUsKQKKjSEBGRvGlOQ0RE8qakISIieVPSEBGRvClpiIhI3pQ0REQkb0oaIiKSt/8PVho9HRYNwW4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_decision_region(X=X_train_01_subset, y=y_train_01_subset, classifier=lrgd)\n",
    "\n",
    "plt.xlabel('petal length')\n",
    "plt.ylabel('petal width')\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OMG, finally to use scikit-learn for logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=100, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=1, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression(C=100, random_state=1)\n",
    "lr.fit(X_train_std, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXlYVNUbxz93BoYdEdxFQdwVcUNU1FzKXdTMDZfUzCzNbFN/ZllZVppaZma5VKa5paXigriSiEmuSIq4sYm4Ifs6M/f3xzgICohsA875PM88MPeee847o9zvPed9z/tKsiwjEAgEAoHC0AYIBAKBoHwgBEEgEAgEgBAEgUAgEDxACIJAIBAIACEIAoFAIHiAEASBQCAQAEIQBAKBQPAAIQgCgUAgAIQgCAQCgeABJoY24GmoYmsrO1evbmgzDE5sehaVq9kY2gyBQFBBCDkdcleW5apPalehBMG5enVOLlliaDMMzpeXb6Oso6Zz/xbUMKlhaHMEAkE5p75Z/YjCtBNLRhWQ2Q2rkeFnScDu82w9FWpocwQCwTOCEIQKykcD7Wj6XTzcvm1oUwQCwTOCEIQKjNfMZmiuati6Yz+HwsVMQSAQFI8K5UPIiyyFgmgHB9JNTQ1tikEYXKsWqYlZZCUqSLyciFJS5jqvMFNgVtMMhYnQfoFAUDAVXhCiHRywqVkTZ1tbJEkytDkG42Z6FpIKrCtZYCrpxFGWZe7H3SfuZhwWdSwMbKFAICjvVPjHxnRTUxyMXAwAapqbok1UkJyQxv3UdAAkSaKyfWW0GVoDWycQCCoCFV4QAKMXAz217JSY31aDRpN9THw3AoGgsDwTgiB4iF0NC9BouB+fSLqcbmhzBAJBBUIIQgnhe+AAjdu1o0GbNnz1zTcGtaWGwhTz22rSEjLJkrMMaotAIKg4CEEoATQaDVNnzGDvH39w4Z9/2LhtGxdCDRsGalfDItunoJE1T75AIBAYPRU+yuhp6NGjB4l37jx23LZqVQ4dOlTkfoNOnaKBiwsuzs4AjBwyhB179tCsSZMi91kS1LJTEhMPmRrhVBYIBE/GqAQh8c4dTlap8thx9zxE4mm4cfMmdWrXzn7vWKsWJ06dKlafJYWcLKHNVLP1SBBDu3kY2hyBQFCOEUtGzzi1HRVUytJAQgJbd+wnVh1raJMEAkE5RQhCCVC7Zk2ibtzIfh8dE0PtmjUNaNEjqFTMNK+lS4h3Lp7QDJHmQiAQPI4QhBKgXZs2XL56lesREWRmZrLpzz8Z2Levoc16jFahMSTNtCAkBLaeChWzBYFAkAshCCWAiYkJ3y9cSO+XXqJp+/YMHzyY5k2bGtqsx/Ca2QwPgmi6pwqaeDsxWxAIBLkwKqeybdWqeTqQbas+sZDQE+nXqxf9evUqdj+ljVet0/jQHVe1J0F7IIQ9hBBK55Z2otiOQGDkGJUgFCe09FnEwwO4248Qk0ACzsXj6hpPEzPDhsoKBALDIZaMjI1GjeDIYVi/PvuQq9qTpD39hG9BIDByhCAYGx4eeHVLzuswTe/2E74FgcCIEYJgrMTcgKCgxw6L2YJAYLwIQTBG9LOEsLD8TovZgkBghAhBEOSLmC0IBMaFEIQS4JU336Raw4a4duxoaFOejpgbT2wiZgsCgfFgMEGQJKmOJEmHJUm6IEnSf5IkTS+LcWW54PdFYby3N75btxa/o7LEwwOvWqdh4YJCNRezBYHg2ceQMwQ18J4sy82ADsBUSZKaleaAK39VsWS5WbYIyDIsWW7Gyl9Vxer3uU6dsK9cuQQsLGPGjHmq5mK2IBA82xhMEGRZvinL8ukHvycBF4HaBV9VnPEgKVli4zbTbFFYstyMjdtMSUqWSmSmYCw8Ols4FB4qxEEgeAYoFzuVJUlyBloDJ0pvDHh3agYAG7eZsnGbKQDeL2Xx7tQMjLoW/cIF0K37g63LhUO/yzkoCDJcrhHXJJQYh1B6OIudzgJBRcXgTmVJkqyBbcDbsiwn5nH+NUmSTkqSdPJOQkIxx3ooCnqMXQy8ZjbT+RKKiIcHtKriQtO7/bhz1Y6tp8RsQSCoqBhUECRJMkUnBr/LsvxnXm1kWV4py7K7LMvuVStVKtZ4+mWinOT0KRg1+exJeBpc1Z5khDYhJAQOhQtREAgqGoaMMpKANcBFWZaXlPZ4OX0G3i9l8e+hZLxfysrlUygq3hMn0rFXLy5duYJj8+asWbeu5AwvC8aM0YWg5shvVFTEbEEgqLgY0ofQCRgLnJck6eyDYx/IsrynNAaTJLCxlnP5DPTLRzbWcrGWjTauWVNCVhoOr27J+BR/kpCNq9qTs6HXCEH4FgSCioLBBEGW5QCgTFfvXxufiSyTffPXi4Ix+xBKk1ZVXOCuCyHxgWy9F4qrKyK9tkBQjjG4U7msefTmL8QgBzE3Cr1R7WkQvgWBoGJgdIIgyAcPD7xmlt6+wEd9C2Knc8UnIz2D2zdvk5qSamhTBCWEEATB4+SRFruk0M8WAs7Fi9lCBeW/s//x3oT3aFe7HQM8BuDh6MGkFycReDjQ0KYJiokQBEEuvLol6yqqlSIiEqnism/HPsYPGE9Tt6b8fflvgqKC+PfGv/Qa1IuZk2byy3e/GNpEQTEoFzuVBeUIDw84cqFMhhKRSBWL6PBo5rwxh193/4pra1f8/fzZ8MsGoiOicXRy5L1P32PR3EU0b90cjy4Pd70/2q5FyxacP3c++/2oCaPo2qurAT+ZQI8QhBIgKjqal994g1t37iBJEq+NG8f01183tFkVgkcjkTq3tKOGSQ1DmyXIg42rNzJkzJBsMfjxhx/pM7MPdVvUJfJ8JFsWbqHnwJ6sXb42WxAebRd6PJQ/5v5Bp5GdGLlyJJHnI/lx4Y8AQhTKAWLJqAQwMTFh8eefc+Gff/jHz4/lq1dzIbTiLoNkp8UuRV/CowjfQvln7597GTJ2CAAbftlAn5l9qNe6HkoTJfVa16PPzD5ERUVxaM8hsrKy8mxnXduawZ8P5tLxS7mu2/DLBkN+NMEDjE4QfA8cwGu8Ny2e74TXeG98Dxwodp81a9SgTcuWANjY2NC0USNu3LxZ7H4NxpgxxcpvVFREJFL5JikxCYeqDgBER0RTt0XdXOfrtqhLbEwsKpWK9NT0PNtlZmbi7O7Mvch7ua6Ljogug08geBJGJQi+Bw7w2ZrFdHirI7N8Z9LhrY58tmZxiYiCnvDISM4EB9O+bdsS69PYELOF8kmNWjW4FnYNAEcnRyLPR+Y6H3k+kirVqoAEltaWebZTqVSEnwzHoa5DruscnRzL4BMInoRRCcLy9WvoN6MvLq1dUJoocWntQr8ZfVm+vmRSTyQnJ/PSyy/z7ZdfYmtrWyJ9GpQjh8t02Sgnj84WRM0FwzNkzBA2rt4IwKgJo/Bd6Mv1M9fRqDVcP3Md34W+2NvZM9h7MEqlMs92yTeS2f7hdhp3bJzrulETRhnyowkeYFRO5fCoSEa0GJnrmFMLJzZFbS5231lZWbw0bhyjhw1jiJdXsfszOGPG4LV+PT5htZ+qTkJJ46r2JGgPJAFx/fYQgnA8Py2PRvkUNarn3r177Nm2h4O+B7GvYU87j3YELgtkS8QWHJ0c6d6lO6u+WcWWQ1uyr9GPs2HZhux2QwYP4fyJ8yzasghHJ0den/K6cCiXE4xKEJzr1CXifAQurV2yj0Wcj8C5Tt0Crnoysiwzcdo0mjZqxLtTpxbXzPJDo0ZQggnvikq2Ht3tR4hJIAHn4nF1jRd5kQpBXtFARYnq+f6r79mzbw/e33mza/4uzKuYcyzwGP0H9Oe1t15j629bWfXNKpauW0q9RvVyXdu1V1dxw68gGNWS0dQxE9nz9V6unbmGRq3h2plr7Pl6L1PHTCxWv8f++Yd1mzdz6O+/adWlC626dGGPn18JWW1APDxKLb9RUXm0fKdwPBdMzigfSSEVKqpHlmXkR/LB/7HhDwZ/Ppj2I9vzwbEPaD24NQoTBWu/X8vHb31M3Xp12Xt6L57dPUv7IwlKEaOaIfR54QUAln+3hk1Rm3GuU5ePJr6XfbyodO7YEfn+/ZIwsdzhNbMZPgvLZqNaYdGX7xSzhSdz+cJlkn9I5syeM2SmZlKpRiU6eXfi+uXrudplZGSwc+NOfl/5OxfOXkCSJNzauTFm8hj6D+tPcmIyzu7OAKTGp3Iv/B7J95LRarREhUcReT2Su7fu6pzKggqLUQkC6EShuAIgKB/o/QshCN9CXvz2w2/cDL9Jo+6NWHh2IdYO1twMu8mOBTuICovi/OnztGjTgpTkFCYOmojSRMnbc9+m0/Od0Gq1HPE9wqrFq/DZ7IOVtRXhJ8ORNTI/v/Iz7Ue2Z8gXQzi46CB/7PuDbeu28XK/l5mzcA6DvAcZ+qMLioj06NSwPOPesKF8cknu4moXa9Wiaf36BrKoYnDx6lWaxsQU+frsGcLMWSVkUckTYhKI0i5e1Fx4wLFDx5g5aSbvfvouq39cjamtKclxyVjbWxMfFY+ZqRmRYZG4tnclIjQCrazF/Tl33Fq55UorMXzscLat3catm7e4m3KXe1H3GL96PCpLFRunb8S+kj2gCy/t9kI3vv30W1ZvX03c3bh8Hdkl5eQuLGU9Xnmkvln9U7Isuz+pndHNEARPT/ayUVCQQSOOCkLMFnKz5ts1vPvJu1SpVgUzKzM6TOiAg7MDt6/c5sC3B+jg3QEbfxsUFgoS/0lk8s+Tib8Zz5+b/mTYvGGM7KhLK7F64WpGjRrF3GlzadyiMfej7rPxrY2YKE1wqOPA8C+GZzurdy7cyfMDnmfBBwvQmGvydGQDJeLkLiwl5VQ3FsQMwQgo7gwBgPXr8Wn0XrkVhJwY+2wh4X4CzzV8jhNRJ5j+ynQ8p3lSr7Uu8ifiWgT3b94nYFUATV9oyu4Fu6njVgdTU1MAOk/qTOWalXFycQLg+pnrBC4LxBRTTh0/xU9bf8LN3Y3J3pNz9atv67/Yn6N7j/LBvg+o714/17nAZbr02HldF7gskJ82/lTi30V+dpbWeOWVws4QjCrKSFAMGjUy6Ea1p8HYI5ES7idgZ2+HuYV5gakjLBwsyEjOwNHNkXuR97gXeQ9nd2cyMzOz2+vTStR0rElyYjK16tQC8k9dcevmLZChWr1qj52LjojO97rSSl1R1uNVdIxKEC5fvcr7H31Ej4EDeWHwYOZ+8QXRN24Y2qyKgYeHQfIbFRUPD2h6tx+aeDsCzsUb1S5nO3s74uPiSU1JLTB1RMqdFMxtzIk+F41DXQcc6joQfjIclUqV3V6fViI6IhqbSjbZN9L8UldUq1ENJLh9/fZj5xydHPO9rrRSV5T1eBUdoxAEWZaZ9fHHdOrTBxOlktnvvMOMadOIT0igZZcuLP7++yL3HZ+QwA+rVxf5+m9XrCA1tfglCI8EBBB44kSx+3nWMMbZgq2dLe2fa8+uLbsKTB1xftd5XNq7EOIXgseLHjTu2JjtH24n+UZyrrQS/Qf35x//fxg2fhibf9Ht6s8vdUX1qtXp2K0jfov9Hjs3asKofK8rrdQVZT1eRccofAjzFi5k1759+G7din3lyrnORUVH88KLLzJj2jReffnlp7YpPDKSASNGEHL8+FNfC+Ds5sbJw4ep4uDw5MYF8MlXX2FtZcX706Y9dq5EfAig8yPEtIFatWHMmOL3V8YYk28h6GgQ08dOZ+OBjURci3isQM3+XfsJPRuKa3tXIsMiUavVuD/nTsvWLXNFGQ0bM4xNqzbh0siFFu4tmPnqTKrVqUYTtya5Ct1Y2ViRHJ/MlZAruD/njmdXz3yL4JRG1E9BfYooo8L7EJ55QUhISMC5ZUtCAgOpXatWnm3Onj/PgJEjuX72bLZzrbCMfOUVduzdS+MGDejZrRtff/YZX3/3HVu2bycjI4MXBwzg09mzSUlJYfiECUTHxKDRaPhoxgxu3bnD+x99ROMGDaji4MBhH59cff/vk0/Y6euLiVJJrx49WPTZZ9y5e5fX332XyGjd1P3bL76gdq1adOjZE6VSSVUHB5YtWEAXz4c7RktMEACCgvAJa1whBQF0LhCbfnsAnvlIpC2/bGHR3EWMfWMsQ0YPoXKVylwJvcLGVRvx3+fPzzt/pkmLJqSlpjH5pcmkpaUxcfpEOj3fCVkrc3jvYdZ8u4Y69eow5OUhrF65mub9m7P106006dSEtDtpTH17Kgn3E/huwXfE343n5W9eplq9avgu9C2zHEV5RRKV5fgVARF2+oBNf/5Jz+7d8xUDgFYtWuBcp46uVkLfvk/V/1effELIxYucPXoUAL9Dh7h87RpBBw8iyzIDvb35+9gx7ty7R62aNdm9RZf4KyEhgUqVKrFk+XIO+/g8NkO4FxfHX7t3ExoUhCRJxCckADD9f//jnTfeoHPHjkRGRdF76FAunjjB6xMm5DtDEDzEmHY5D58wHDd3N9atWMfgToNJSUqhRu0aDB03FJ8gn+zaBhaWFqzZuQbfv3z59ftfmTFxBgqFgpbtWjJ19lR6DuzJG6PfyE6B0aJHC478cgT/4/5MHTEVE1MTWvVvxZuz36Rmw5oAuvQYyzaUyQ05Z3oO4GF6jjIa/1nimReEa+HhtHFze2K71m5uXI+MfGK7J+F3+DB+hw7R+rnnAEhOSeHytWt06diR9z78kFkff8yA3r1zPcHnRSVbW8zNzJg4bRoDevdmQO/eABzw9+fCpUvZ7RKTkkhOTi623U9FzI1yvSehMBjLvoUmLZow/4f5zP9hfoHtTE1N8RruhddwXabeu7fusunnTSz5eAkfvvkhyUnJWDe2xr62PZVrVebFOS8ycNZAFvVeBMBrK19DaaIkLTGNY5uOcXT9UW5cuEH3pt3pNagXoyaNwqm+U6l8xvwiibZEbMnnCkF+PPNOZQsLCxISE5/YLiExEXMzs2KPJ8sys995h7NHj3L26FGunD7NxLFjadSgAaf9/WnRrBkfzp/PvIULC+zHxMSEoIMHGTpwILv27aPP0KEAaLVa/tm/P7v/GxcuYG1tXWy7C40+2ujI4bIbs5Qw5kikgjh57CT93PtxM+omC1cvZO+pvbTs0JLI85HM7TSXsOO6FLiPRg7FXollbue5hAWG0XVcVzx7efLj1h9RKBQMfW4ou/7YVSr2ikiikuOZF4T+vXqx+a+/0Gg0+bZJSUlht58ffZ5//qn7t7G2JinHE3rvHj34+fffs5/ab8TEcPvOHWJu3sTSwoIxI0YwY9o0Tp87l+f1epKTk0lITKRfr158M38+50JCAOjVvTvLVq7Mbnf2/PkC+ykVKqj/ID8ejUQy5mI8MVExvDHiDRb/vJj5P8ynZbuWVKlehTfeewMTjQleM7z4fuz3nN13Nlfk0O4vd7PAawF9p/el7/S+XDpwiQlvTKBx88bM+mIW6/etZ9678zhz4kyJ2ywiiUqOZ96pDNChZ0/GDBvGm6+9luf5D+bNI/TyZf5ct65Ido169VWC//uPvi+8wNeffcbSH39k9W+/AWBtbc36n37iyrVrzJg7F4VCgampKSsWL8a9dWuWrVzJ96tWUatGjVxO5ZuxsQwaPZr09HRkWeb9adMY5+3N3Xv3mDpjBhcvXUKt0fBcx478+M03hF25wtBx41AoFKXrVH5Adn6jbt0r9NLRowQFgcrlGmZNQrF3gB7Oz6Z/IT+mjZ7G+bPnsapklR2RpI8WAt1SUtL9JFRmKl7o/wJJKUlER0STmpBKfFw8terVeuw6fWRP1LUojh85zvJNy4tt56ORQ3mNJ/wHDxFRRjm4ev06XQcM4OURI5j++utUr6bbRRkRGcmCpUs5+PffHN2zh2pVq5aa7YakNAQBqFDpLIqCMYWpgu4mO3noZF796VXaDmjLoTWHOLbpGMPmDaN289qEBIQQ+Esg7l7u/DX/LxzqOtBpZCd6TOzBZy98hkqhYvans4HH8xX5LvRl3PhxzHp1FgHXArCxtSmWnSKq6OkQUUY5qF+vHsf37WPe11/TxMMDF2dntFotkdHRjB0xgsB9+3Cwtze0mYJyhqvak7Oh1wghlBiH0HI9W4iPi+cf/39IS03D0ckRlZmK65evY6oyxaOzB1VrPHzYuRh8kbD/wlAoFbTt2DY7HcWGXzagzlLTpn8blCZKLh2/xODPB2Nd05qE+ASa9miKfW17Di49SEp8ChN/m0jAqgB6Tu5JZmomff/XN7vwTl5RP38t+4tK9pWIvxdfLEEQUUWlh1EIAkAdR0dWLV3KonnzCLt6FYVCQZOGDbGysjK0aRWbI4ef2RkCQKsqLnDXhZD4QLbeCy13s4WkxCS+nPUle//cS9uObUlPS+f0P6fRaDS0dG+JfVV7Ppz6IV17d2XA8AH8uPBHYmNicfd0Jyszi0+mf0K7zu34aNFHug1mdlYkxCbgUMchO7fRnet3AFBZqKjdoja3r93G3NocZ3dndny0AwDLSpZYVrLMXlrKK+pn0/VNJMQlYFOp6GIAIqqoNHkmnMpPs+xVqVIl2rVpQ9tWrYxCDEp1SVDvXF6/vvTGKCe4qj3JCG1CSAgcCi8fDueU5BTG9hmLjMz+8/sZNn4Yly9e5rvfv+PXXb8SGxPLcz2fw/+SP1qtljeGv8FLL7/EkdAjfPvbtyzftJy/L/+Nm7sbw3sMx6GqA006NyFgQwBArtxGKpWKzLRMbpy/gUatoXmv5tk5kQDaDW7H/hX7C8xXpDJR0bJdS+zs7Yr1uUVUUelR4QXBPCuLe4mJpXvjq6DIssy9xETMs7JKbQyvbmW8B8KAtKriQtO7/bhz1Y6tpwwfibRi4QrqNazHFz98wanjp5g+djrWla354/c/OPXPKZwaO/Hx2x8zduBY/Hb4YWljyfKFy/lx0Y9M9p5Mf8/+vDvpXZq3bs7o10aTej+VjLgM/Fb4EXk+Mlduo0p2lbh46CJb/7eV+Jh4XLu5ZudE0qg11GlRh4t/X8StpVueUT875+0kNjKWCW9NKPTn8/fzz7Zzsvdk/P38gbyjijbN3MT9uPuPtRU8HRXeqZylUBDt4ED6U6acMBbMs7JwvHcPU622dAYICsLniHWFzW9UVM7eNWwkUmZmJl0adGHj/o1ERUQx73/zMLMzY8aOGbmcwYEbA7l55Saxl2IZs2gMv077leoNqzPyi5E06dgk2yE74ZUJzJ48mw++/oDffvqNi6cv4tLEhef7P8+Vy1eIuBpBalIqt6JuUc2xGs1aN3ssssfdw52Vi1cy4pURuDRyYd/ufURei0QpKbl94zavvPUKr894vVCf70mO45xRRhbmFqhN1bz48YvCyZwPFSLKSJKkn4EBwG1Zll2f1D4vQRCUAyp4fqPiYKhIpNDzoUwfM5195/Yx2XsytxJv4fGiB51Hd+b78d9nF7q5cvwKPp/50Hpwa5JikwgLCOP5t56n1YBWjxXBUUkqeg7sycCRA/nv7H/8suwXfP/0RalUotFo6DOkDxOmTaB5q+b52hVxNYK1y9fy1+9/oVFryMrM4rnezzFu6jg8uxe8Oz8nT1PYRhTBeTIVJcroV+B74DcD2yEQFAlDRSJlZWWhMtPVLYiOiMba0RoTM92fc05nsCzJyLKMg5MD4UHhqDPVVHGpQmZmJqkJqQRuDiTozyAiz0ViW8kWWztbeg/uTfNWzVm0ZhFf/fQVqcmpWFpbYmLy5NuFU30n5i6Zy4eLPiQlKQUzC7Nc9RUKy9M4joWTueQwqA9BluW/gThD2iAoIWJuGIVzOS8M4VtwcnEiOiKauLtxODo5Ym1vnZ1SIqczOOZ8DA5ODvy37z8q165MVnoW6gw1ty7eYna72YQFhuE+2J2WHVqi1WgJDQ6lb5u+hF8JB3QpVGztbAslBjlRKBTYVLIpkhjA0zmOhZO55Cj3TmVJkl6TJOmkJEkn7zzI+CkoZ3h46JzLMcZdfa4sI5Fs7WzpNagXv//0O6MmjCI+PJ7jm48THxuf7QyOuxrH6b9O07xXc8ICwrCwsqCqc1X8Fvux4e0NjF86nr7T+3LV/yru7d2p7VSbbQHbeOWtV5gwYALJSYYLGHiadBQidUXJYXCnsiRJzsAu4UOo+PgsvAAzZxnajHKB3rdQmllUo65HMazbMGbOn4l9VXs+n/E5N6Nu0q5bOyrZVGK/z35kWcaqshVJ95JQZ6ixdrDGxtIGlYUKCxsLrGysSLybyLXQa7Tq2IqpM6fStVdXpoyYgmd3T8a8XvJ+ocIWrHmawjaiCE7BVAinMghBeKbQV1R7xvIbFZWyiEQKuxDGNO9pmJia4DXCi5OBJznqdxSNRkPdlnWp0agG5/aeQ6lS4jbAjdNbT5ORmoFnd0+c6juxa9suZGRe/fFVbKvaZkfnmChNWPjhQnYc31Gi9oq0E4ZBCILAMDzj+Y2KQmnPFrRaLf5+/mxbu43UlFSuhF2hfpf6SJLE+UPn8ZzgiWNLRxJjE7Gvbc8Pw37AwtKCyg6VaT+6PX2m9cFEpfMR6KNzPlvyGQM8BhAUFVQoG5ISk/jvzH9oNBrqN65Pjdp5f84nRQRFh0cTcTUClZkK1zauWFhaFP8LElSMKCNJkjYC3YAqkiRFAx/LsrzGkDYJBCWNPhIpgFDsHeJLdLaQkZHB8i+Xs2nNJmo61sS2ki2xUbFYhlkycOZAzvie4eiqo1jZW6GyUBEfE49WrUWWZKwrW9N3el+UJsrs/vTROfH34wu1k//+vfssnruY3Vt307BZQ0xVplw8dxGPLh68++m7NGrWKFf7/CKC1v63lnH9xnEh+AKNmzcmLTWNiKsRDBk7hOkfTcfK+tnPKlAeMKggyLLsbcjxBaWEvniOmCVk82hOJHsHqFWzeHsX0tMzmDzkNcwtzNm4fyMujesjSTBpxCTsmtux1HspSlMlwxYNw22AG0m3k6hSpwrzPedz5+odJK1E5PnIXE/r+uic7b9vp+fAngWOf+/OPUb2GEmn5zvhe8aX6rWqA7qUGlt+2cLoXqNZs2MNbm0fVizURwTlHPPwz4eJCY9hyntTWPnnSszMdYWqoq5H8c2n3/By35f5be9vQhTKAIMvGT0NYsmogrB+PT54GeVGtcIQ9GAVxqbfHoAiLSVtXG3D6m+HERtzAjMbM2S1jNItJvIkAAAgAElEQVTMDK0mCwtLc9KT0zG3NScuJg77uvZYOViRkZyBlb0Vty7dQp2sRtbKNGjbgEGfDMq1nu81wItvPv6GLUe2sHvbbv7Y8AfJicmYKE2oUr0KoLuxJ91NokXbFsz+anaeTt301HS+mv0Vc7+Zy6a1m/LcVXz136t8Pehr3p/3PpPemfRYP97jvfHZ5INDNQc+WPBBif47GBMVxofwNAhBqCAY8c7lp6UoO51lGSZ4rSTwyCKGfT2CxFvxnN97Hq+PvXBqW48bwVGsGLqCKtWrkBCXgKmVKYM+H0Q1l2rcuXaHf9b8Q1OXphzcdZAGTRpQuWZlYqJiqFm7JnXr1mX3H7v5YMEHREVE8ef2Pxn8+WDSk9I58uMROr7ckUbtGnE37C5LRyxl6fqlWFpZ5ukonvzGZOa9q0upMfLrkdnnNs3chI2VDSlJKUiyhJmJGduObsvX4Tz0paF8OetLjl45KnwKRaSwglDu9yEIKihGviehsDxavjNWHfvEayQJwi7/ik01G9oO9+TktlMM/nwwDTo3QKGEai7VUKqUKMwUNGrRCPta9nQf2p3mbZrTbWg3RnwxguS0ZKbMmsKlkEucPHKStPg0/j3yLzfCb7B843IGeQ/ijw1/6Prt2IATG0/Q/6P+NH2hKYmJiaTGp9KwY0O2b9meqz6B0kSZXZ9g468bMTU3pbJT5VznRi4cSWX7yuwO3E09l3qMfWMsQL79+O3xw7mBM8Eng0v7n8PoEYIgKHn0voOFCwxrRwXBwwOa3u1HUqTEF7N+ZuTAkYzvP54vZn7BtUvX8rwmJSkFK3vdmnpGUjp13Z2QFBIyMlkZWVjYWpCSlELc3bjsCCI9dVvUJToimm59u1GrTi32B+9nzfY1+F/y56dtP+HeSfcgmZyYjLO7MwBxUXHUblEblaUKrUZLVkYWdjXsiI6IztdRHB0RTVJCEuY25nmeA8hMz8TCSvfUX1A/llaWZGZkFuHbFTwNhRIESZKUkiTVkiSprv5V2oYJKjZeM5sZ2oQKxeHD6/lm4igi/bNo2q03Dfq+QKZpJt49vfn4rY/RaDS52tvY2XD32l3SEtMwszEn8mQEslZGQsK2mi2p91OxsLDATGWGlV1uZ6zecRz8bzDODZ2pUr0K9RrVe6xOgbWtNeEnwwGwr2PPjfM3yEzNRKFUUL1+da6cuELturULTB2hkBSoLFR5ngNwbuhM8L+6J//8+qnlWIvQkFCc6jsV7csVFJonCoIkSdOAW8B+YPeD165StkvwrCBmCU/k3393sXbtLBYsCGD+/LUMbf4ZDZu+gOuQkaw4vYLLoZf56n9fZbeXZWjYYDwKEyV+C3fi/lJbtn+4nSsBV9BqIPJsJKYWptSuWZv4u/Fkxmc+ltZh5Msj+W3Fb4ycODJfu4aNGqbr9/gV2nu3Z/dnu7l44CK2trYoTZWkxKXQslXLfFNH9B/cn4R7CdwNu5tvWokRE0bwx9o/SE1JzbefevXq0bRFU+q6iOfQ0uaJTmVJkq4A7WVZvlc2JuWPcCpXPEQ6i4KRZZnp01sxbtwC2rbtw6nTvuw7vJzbd8IxN3emw5h21G3ehMUjX2Xf6X3ZoZ0bV9vw1+8fcObEz1jYWqBUKnVRRuoszCxsqOvYl/P/rqVxi8ZM/2g6iz/fQmJCJE1cazPy5ZEc8DnA9SvXWbd3HUqlMl/7vv/q+3yjjJo1b8bmNZv5ZdcvxN6IzRUdNPClgfy89Gc6Pd+Jtp5tC0wrMfv12cTeiGX5puX8e+zfXG3btW/HT4t+4qetP9GmQ5vS/cd4himxKCNJkg4DPWVZVpeUcUVFCELFQwhCwYSFBbFkyRh++CGUM2f92HHkM3q/1w/HZk5EX4hg3+I9tHr+JUJPH6ShqwOffPhJ9rWyDKeOn+RN7zdp3LwxXiO8sLWz5feV1/nH/3ccnRqQmXEZpbIuGu3LdOxqjXPDYDav2US9RvVY9vuyYtc33rlpJx9P/5guPbvQc2BPVGYqTvifYMfGHQyfMJwZn89AoSh4ISIrK4u50+ZycPdBho0bRst2LUlNSWXP1j2cOXGGRT8vomtvkdaiOBRbECRJevfBr82BxuiWijL052VZLvM7sxCEiofPwgtGV03tafD338iJE9uZOXMzXyz2ouv7HXByc8k+HxF8Df9F/+BgPZhYeRPdXx6OWz1z3Oq5Zd/MM9Iz2PvnXg7tOUR6ajqOzo6YmLzCiaPtkeUsEuN9sKm0jRq146lZuwZDxw3Fzd0tP5OemsT4RLat28bJYydRq9U0at6Ika+MpLZT7afq52roVTb/spmIK7rUFZ1f6IzXCC8srSxLzFZjpSRSV+gfHSIfvFQPXgAVZ/OCwKB4dUvG58gN3W4ssXP5MVQqc9LSkgC4fSccx2Yjcp13bObErdsb0abt5ML5E1w79R8qSxXJ9+/Ss39PpsycQv0m9Rk8ajCDRw3Ovk6WYbAnSJIplSoPYXtgeySpdD6DrZ0tE6ZNYMK0wtdLzov6TeqLzWcGJt+5nCzLn8qy/ClwQf97jmMXy85EQYXGwwOvWqcNbUW5xdW1K6Ghx4mPv021qs5EX4jIdT7i3FXu37pNSMgRvL0/Ye2qGCaNDue9jatJt6nE8OeHczbobK5rZBnWfFsp17E131ZCvxhQgfaiCsqYwoSdzi7kMYEgf44cNtqKagVhY2NPp05D2bDhYxxspvDnvD1EBF9Do9YQEXyNn99cjayxQ6m0R6OZjizrJlptM0ciqT+h6XPzeG3Ea2Rm6mL09WLgs8Uar+HJjHglkWo11PhssWbNt5XQanXnN64unu/gUfz9/JnsPZn+nv2Z7D0Zfz//Eu1fUDbku2QkSVJfoB9QW5Kk73KcsgUM7mAWVCDGjMErKAifMEMbUj4ZP/5r5szpholJBvEJk9j6v23I8iY0mprcv3kPU1Nzevf249gxJQoFDB4M27fD5aPOODtPw67mGuYt+YU33x9EDZMaWNlo8RqezCvTE/h5aSVu3TShek01VtZafl76UCxkmRJZRsor5cSPC38EEDUOKhgF+RBigFPAwAc/9SQB75SmUQKBMWFtbccXX/izadM8fH3fJzHBCUmyJSPjOKam5ixZcow6dZqyfTscPap7AXTpohOHAwfewv/QJgJ6dsXVNR7vV5tk3+wnvq0rO7tzszWbfrYFwGt4MhPfTigxn0LOlBNAdsqJDcs2CEGoYOQrCLIsnwPOSZL0uyzLWWVok+BZJUY4l/PDyqoSI0Z8RGpqAocO7QfUmJg4Ym2dyLZtX2JmZkX79oOQ5V5Ikm6ld/Bg3U3f2roy5mkWupxI7CGE0OwMqnpR8NlinT1WSYoB5J9yYkvElpIbRFAm5OtDkCTpvCRJwcApSZKCH32VoY2CZwG9c1lfK0GQi82bP2f0aHv+/ns75uYdkCQLsrLCiIuLISzsX2rUaMCyZbOJjGxOZuYFQLdsJMsQHh5MtWpO2TmRNPF2BJyLJzQj9IkO5pKgoNQVgopFQU7lAYAX4PvgNfrBay+wp/RNEzxzGPFehEdvwDnf+/gsY8OGubRv/z3Vq9/BwSGLdu1aMXZsMqamrbh58xr79u3Bzu407dv/j6SkF2jdOpK//4Zt2zLx81tNjx6vZPfnqvYkYVc/zp+H6e9nsGGdCe1fusb2wBt4DU/OdjCXlCjkl3JCn55CUHEoaMkoAkCSpJ6yLLfOcWqWJEmngf+VtnECwbOAry+kpT1c4pFl3dO9hQX06QPr1n2Al9fb7Ns3BbX6b6pXv8D77wfj56dCrV6ALI/m5s1DNG8eRoMG40hJCSU5eQF2dt/i6zsBSfJg48ZWzJkDCgVotbB1qy8pm1aQmRmOUmXJwW1JHNiSQYP6LjRvNQkrm+4ltmyk9xNsWLaBLRFbcHRy5PUprwv/QQWkMCU0JUmSOsmyfOzBG09E2mxBcVi4ALp1NwpfgizrxEDvCNZHCB09qnMKBwRsJTMzjVGjvsLPDzSaldy+PQVZVrFmDchyLxSKpWi1Y7lwoReRkWvw8OiIn583krSfGjUaU7XqJoKDYf58mDMHZs70JUn5Gf1m96ORex8unTpL4K/H6DS+H7bVrDn83Zf0fjEFKLkbdtdeXYUAPAMUJpdRW+BnoBIgAfeBV2RZLvPdRiJ1xTPC+vX4NHrPKAQBHs4I9KIADyOEVq16i8DAbXzyyT5sbGrwyis9keXVQFsATEzAyQlu3hxOWtohFIpGKJXpZGVdpnPnH5g4sRdmZpYsWWJD8APPnqmNFyO+7kDbHi7E3rqCdQ0rbl2K5dDSI4xZOoXoS5fw/96fP/4S+0KMhRKrmCbL8ilZllsCLQE3WZZbGUIMBM8YRuRcliTdzT8ngwfD8ePbOHx4HXFxMUyf7saECVVRKEKANdntNm0C0KJWhwMpaLXnyMq6jiyncPz4q7z5ZlPGj69BcnJnNJrN6B7wwmn1nK52QJY6A5W5itqujtyLvouFiRkujZtzO/wuW0+FEpoRWjZfgqBCUFCU0ZgHP999kOhuIjAxx3uBoGjoncv6avMVlIIcxY8e374997GPP/6UBQuGUaeOrpDQypXX+eWXO2g0TYAf0cV0wIgRWm7dGkdm5mmUyhewsfFHkhRIkjVOTt6oVBYsWHCczMx30Wi+QK1+A1l24uzfuhQYpiZmZKZnciMkGgdHXerq6AsROFZtTEZoE0JC4FC4EAWBjoJ8CPoySyW7x10gALxqncaH7oY2o8g8yVGsJ+dy0cNlopPs2vUpnp4rMTV9lWvXnPn6a28iIwOBnUBrdMF8X6PRSCQnBwJaNJpfSUxshZWVEw4O3kRGzqBSpV+ZNesltNqL1KrVi6ysHkAH9izcA/Sjkbs9oYd0PoTu4/oTEXyNfYv3MKj7R7Sq4gJ3XQiJD2TrvVBcXaGJWZN8P7O/n3+BdQ0EFZ+Coox+evDrAlmW08vIHoExERZWIf0IT3IU50wJIUk6kdCLgSTB9evvYWfXHDe3Vzl1CmR5J2FhbVEoOgIbUCo/R5ZnA5+j1WqAFOzsFhIfvx7IQqm0Q6N5i6ws0GjGo1L9hJnZbszNB+Hmtpjg4NewVC/B97MfOGwejoWFDWTZsO9rX6pVdWZQ949o2+aharmqPTkbeo0QQolxCKWH8+OiINJTGAeFrZh2Czj64BUgy3JCGdj2GMKp/AwRFITPEesKWyuhIEdxXuGcOUXipZfMef31FfTsOQGtVhcddPZsMFlZg4AITEyqoFJlkpqq+zNTqdqgUFQiPT0A6IlKtQ1JMqdqVVCpICnpJ9LSjjFs2G8MGiTz+usNmD17B/XquT715woxCURpF//YbGGy92Q8p3lmp6cAuH7mOoHLAvlp4095dSUoR5REPQQAZFluIElSXaAL0B9YLklSvCzLrUrAToGx4uGBFxU34Z3eUawXhJSUg5w5s5CjR+Ows6vOyJEf06hROwDu3Ink7783kZh4ByurymRlZRIScpiYmEvUq9eKmTNfZMwYN8zMrqPV/oeX13o0mkx27FhC3bquZGTMQqGwJi5Og1b7DpJkDsCyZTBjBiiV1ZHlhAdiJGFnV520tKI9s+U3WxDpKYyDJ0YZSZLkCHRCJwitgf+AzaVsl8BYiLlRIdNi62cImZkRXL9ei5s3exIREYutbRViY6/y/vvtmTTJha++Gsbbb7fm1q3rJCfHs23bV4DM6dP7UCpV7Nu3Cm9vJzSajQAoFM2JivqS55/X7Ty+dy8Ja+uhWFj0Q62ug1r9UEHfeku3CS0rKwylsibbt+vKUcbGXsPevmaRP1urKi40vduPO1ft2HoqlFh1rEhPYSQUZoNZJPA2sFeW5Y6yLPeXZfnLUrZLYAx4eODVLVknCuWEwkQO6cXg8OG7xMQ0x9GxCiNHxuHgcI6WLfeyfPlFVqy4TlzcTYKCfFixIoxOnbwJCtrB//63lQ4dhpCensK5cwdQqXYDe5CkmUyduhE3NwgOhg8/fBdr6wZkZVWmXj0/7O1BlscBPzFihEzVqhAdDZmZWkxNV9KlyziOHoXvv99JrVqNqFHD5XHDnxJXtScZoU0IOBdP835dRHoKI6AwO5VbA52BUZIk/Q+4DPjLsrym4MsEgkLg4QFHLhjaCqDwkUN6R7Esv4GVlRUmJmc5dkxB9+6644sWwT//BKJQtAaCmTXrS27ePEDjxis5dKgXqanNyMhw4vr1SOzsfsPE5DWaNt3JihV9GTJkCFeu/Mr9+/tp3vwvYmOzOHFiCs2aHaF16+cJDbXgyJEPGTDgc3btkpHld6hWzYlJkzqwbt1VfH3f5v33V5XYd6KPRNLUssO1Jxz+7gB3Iu+I9BTPKIXxIZyTJOkqcBXdstEYdHvehSAISg4Dp8V+msgh0AnEqlU+jB27gP37FURHw+HD8O238PPPkJW1AlPT93nhhSPs3bsSqEF09EAsLeH8eUfAh8xML27fngrcIzq6C6amldmwoT6yHEOrVvMwMRlEUhJUr36D0FB3nJwmY2v7FQkJU9m9ezuyrMHMzJKRI79l3brZ7N+/hvHjv6Bt2z55fMLi4ar2RG1Xg8Yz3LF3IM9IJEHFpzBRRicBMyCQB5FG+sR3ZY2IMnpGWb8en5g2Bs9v9DSRQ6mpyYwcacP27RpkWcFbb+mWcB72ZYOJSTSSFEtWVhNUqslI0o/IMqjVurEUiig0GifAAsgAJBSKmtjYbKdKlTYA2NtDXBxkZV0gIeFHZHkfZmbpWFjYAjJpaUmoVBa4u/ejT5/XqV27USl/Sw8jkfQ1FwTlnxKLMgL6yrJ8pwRsEgjyZswYvNavN/hGtUcjhyD/MFKFQud+U6vVqFQqvvsOhgx5eN7SUoFarUaWMwDo0kVNQICuL5Psv7rqaDQKtm1LYtYsBXFxn6PVpuDg0CZ7zDlz4P33QaVqRtWq37F4ccmUvSwO+kikAEKxd4gXs4VniMLkMhJiIDAK8koxoS9C8yjm5paYmVmxdesXaDQwbVru8+npnmg0u9FoFgMOHDrkiyyrs2cIWVmg1e5Gkjrw9tsKtFpISdmFRtOR+HjdmLKs26OgH19vX873hiKvSCRBxcegaawlSeojSdIlSZKuPHBYC4yZI4cNlt/o0RQTixfrfh49mrco+PpC3bre7NjxDePHpxMdDba2MGkSmJmBVjsFjeYrFIpNmJi8gyzXQ6FYS4sW+pt9FgrFQiwsphAVBYmJBzExuYmpaX/S0qBaNahcWRdxZG8PPXvqfv79t84erVb309fXIF9XNjkjkQ6Fh3IoXIhDRcZggiBJkhJYDvQFmgHekiQ1M5Q9AgMzZowuBDXMMDvV8koxMXiw7r2FRe5lGr0DWqNZhlZrRmJifSAEhQIyM6FGDQB74AoKhZIaNcYgST+QmfkBSUlrqF49CoViGCYm1XB1HYZSuYO4OG969FiDi4syezx3d3Bz0/1MT9f5EuztwdwcduzQiVVammFnCvBwthCxTjdj0JfvFFQ88vUhSJI0JL9zALIs/1nMsT2AK7IsX3sw3iZgEFA+YhAFhsGAexL69Hk8D1FePoSH6azN8fe/QnR0FzIz3YiPr8Hvv1dBo4kF7tKkiSdNm3Zk//7W1K7tSXx8d65cmYlWOwk7u+Z4eHQkJKQ5NjbmWFpu5uLF7sgyeHnBiy/qxunTR1cFTX/T//tv2L9f93tBqTIMgYcHoPYkaA+EsIcQQoXjuYKRb5SRJEm/FHCdLMvyKwWcf/LAkjQU6CPL8qsP3o8F2suy/GZ+14goo2cfn4UPngdmznpi26tXz3DhwlG0Wg3Ozi1p0aJbtrO3LJBleO893e8ZGREkJHyFRnMbExNHvvtuDpUrVwMgLS2ZoKBdrF17G6XSARMTJ7y8QgAt9eq1onHjjrz//sO7en6O45zjFdSuvJBfXiRB2VPsKCNZlieUrElFQ5Kk14DXAOpWrWpgawSljdfMZg9FIR+uXw/mhx8mExcXQ7t2A1AqTTh48FeystJ59dWlpRKH/yg5HdCyDKmpTpiarqBqVd1N+siRh0/v5ubWxMWNxM7u4fUZGZ2zi+bk5ch+9Mk/P4d3eZohPIqrmC1UOAoTdookSf2B5oC5/pgsy/OKOfYNoE6O944PjuVCluWVwErQzRCKOaagghMefp65c3sybtxXdOv2MiYmSgBkWebs2YN8880Y3nxzFR4eXiU+tn45SX9z9veH557THdu5U9emSxfde33o6qBBD9f79Us8Ofc6yDIEBOR97tEd0/n1UZ5FwcMDuNuPEJNAAs7F4+oaL2YL5ZjCJLf7ERgBTENXU3kY4FQCY/8LNJQkqZ4kSSpgJLrqIAIBLFyQ5+GVK6cxevRnqNUT8PFR5nCoSoSHv0CPHn+xfPlrZGVllqg5vr4Po40kCa5d04WOWljoXl5eUK8eXL+e2xmtUBTsrLa0fLIj+2kc3uUVV7UnSXv6ERKCCFMtxxRmhuApy7KbJEnBsix/KknSYnTlnIqFLMtqSZLeBPYBSuBnWZb/K26/gopPfstGkZEXuHHjEj16jGf37vzSTHSkdu0mnDixnc6dh5eIPY+mtRg0CFJT4fZtOHlSt3ls+3a4fx9atHhok/5G/SRndWEc2YV1eJdnxGyh/FMYQUh78DNVkqRawD2g6Ll1cyDL8h5gT0n0JXgGWb8+V/GcsLATtGrVE5VKlb3+fvTowxu1/glalvtx6dI/JSYID6OKHo4ny7qQ0Lg43U5i0C0f5XeTzitSqTDnCttHRUL4FsovhQnJ2CVJkh3wNXAaCAc2lqZRAkF2WuwcG9VkWUaSdP9lc96k9ehvxgqFEn30XHp6Cvv2reL99zswblxNXnutPqtWvc2NG0+33+HR8SRJNzOQJFCrbxIX9yn79zdj3LgavPVWS/7882sSE+8V7cMbAR4e0PRuPzTxYt9CeaIwgrBQluV4WZa3ofMdNAE+L12zBEaPhwdetU7nOlS/fhuCgw+h0WgKTDNx9ux+6tdvw+3bEbzzTluCgnbi7f0JS5ac4sMPd2JmZsmsWZ04ePDXQpvz6Hj6tBIpKQeJimqJWn2LVq3WsmTJaV5/fTkRESFMm+bK5csni/ElPPsI30L5ojBLRseBNgCyLlNXhiRJp/XHBIKywsWlFQ4OtTl6dDP374/KM+omLu4cV6+eYubMLcyY0YFevSbx4ovvodXqHLwODrV4+eUv6N59PB991I2qVZ1wde1Ozu0L+rbwcEOYvv/OnXXjzZ8P585dQasdybx527h8+TkCAnRtBg+uRZMmnQkK2s7nn3vx3XfB2NpWrbBLPKWN8C2UHwraqVwDqA1YSJLUGl2EEYAtYFkGtgkEuvxGYWHZvoRJk5Yyb15/OnWyonPngQweLGUv59y9e4oDBwbz+uvfcubMPmxs7HnxxfdYvFjnBJ4zR3ej12rh118bUa3aAhYvXkCDBt1znZs/Xxf907z5w4I5FhY6MZAk8PODdu0gIuI7HB0nc+PGc0iS7ryFhS4i6eRJaNduMO7uu9i3bxWmph88VmhHkBvhWzA8BS0Z9QYWodsfsARY/OD1DvBB6ZsmMHr0+Y1y0KiRB3Pm7CA4eBaHD7fk998/ZOPGT5kzpxsBAV5MmbKIrl1HcfjwOnr3noxWqxOD4GDdjV5/ww8OBkvLYSQmBnHuXOxj51JSdNfpk9v17v1wf0FaGvTqJZOc/BtTpryWHYEkSdCrl04MgoPh33+hZ8/X8PH5rdzkHSrvCN+CYSlop/JaYK0kSS898B8IBOWCpk09+eGHiwQHH+K//3SpK/r3f5P27QdhYmIKwP37N6lZswEKhW5moL/Rjxih68PNDebMMeett2pjY3OL4OAaj5zT3eD1IvBoJFNmZjqZmWlUq1a3wAikFSsakpQUy5AhFS9M1JCI2YJhKIxT+ZgkSWskSdoLIElSM0mSJpayXQLBQx6JNgKQJImWLZ9n1KhPGD16Hp06Dc0WAwArKzvi43UOSr0o5GTOHJBlDQkJt5k+3e6xcwpFwZFMKpU5CoWCpKS4AiOQNJqbKBSVhBgUATFbKHsKIwi/oNs8VuvB+zDg7VKzSCDIiYeHbtnoyOGnuqxTp2HZUUT6paCczJ8Pp075Uq2aM6tWOT12TqstuGCOJEl4eg7l0KG1+UYgyTIkJv6KtfXwfAvtCJ7Mo5FIh8JDhTiUEoURhCqyLG8BtKDbYQxoStUqgSAnRaiz3LXrKC5fDuL48R3Zy0VubrB5s+7nuXNxLFo0E43mncfOBQfD55/DX38VXDDHy+st/vxzIb/9diW73aJFupoFwcFgZnYarfYXXnjhjXwL7QgKh362kLSnHzePNSEkBA6FC1EoaQojCCmSJDkAMoAkSR2AhFK1SiDIi3zyG+WFubkVc+bsYMWKycTGzqJx43DmzAG1Oh1Pz3VIUgeqVh1EnTojs30G+qUlNzewsnpynqGGDdsxevTn7N37HA4Oy+jZMwGFApo3v0PVqgu4fr0Pb765irFjXSpU3qHyjIfH4+U7xWyh5Mi3HkJ2A0lqAywDXIEQoCowVJbl4NI3LzeiHoJx47PwQqHqJOTk9u0IfHyWcujQWjQaNZmZ6bRo0Z3+/afh4dEfyL3v4NH3OfMH5fUeIDT0H3x8lnLixA5MTVVoNGo6dRrOwIFvU6+eW77XCYrP2bvXMGsSir0D9HAWexfyo7D1EJ4oCACSJJkAjdHtRbgky3JW8U18eoQgGDdFEQQ9Go2GtLREVCoLVCrzJ19QBNTqLNLSkrC0tEWpLFRmeUEJIYrxFExhBaEw6a/NgbeAz4BPgakPjgkEZc9TLBvlRKlUYm1dudTEAMDExBQbG3shBgbAVe1JRqjwLRSXwvgQfkNXHGcZ8P2D39eVplECQV54zWymy2/0SAiqQADCt1ASFOZRxlWW5WY53h+WJKngGocCQWkSFlakyCOBceCq9uRs6DVCCCXGIVT4Fp6CwswQTlkO21EAABNaSURBVD+ILAJAkqT2gEjhKDAMjRrpNqqtX29oSwTlGDFbKBqFEYS2QKAkSeGSJIWjy37aTpKk85IklXmkkcDI0W9UEwgKgfAtPB2FWTIS+RkFAkGFpVUVF7jrQkh8IFvvhYpIpAJ44gxBluWIgl5lYaRA8Bhi2UjwlIjZwpMpzJKRQFC+8PDAa2YznSgIBE/Bo74FUaEtN0IQBAKB0aGfLQScixezhRwIQRBUWLxqndZtVBP7EgRFQMwWHkdsqTQyZFnmcHAwvx0+zM3797GzsmJ4584M9PDA1KSC/XcYMwav9evxobuhLRFUYPT7FgIIxd4h3qj3LYgZghFx6/59PGfO5O3Vq2nboAFvDxxI79atWbpzJ02nTOFiVJShTRQIDIKYLeioYI+EgqKSnplJn08+oX+7dnw2ejRSjtSbr/TsydqDB+k5dy7/Ll5MTXt7A1paBPTFc8TuZUExeXS2ANDM0XjKdwpBMBK2BARQtVIlPhs9muffe4/EhDxKWqjVfL97N/PHji17A4vKmDF4BQXhI9JZCEoI/b6FoD1g6RlIwL14XF3jjWLvglgyMhJW+fkxbcAAJEkiMSGBk5UqPfayUyhY7eeHVqs1tLkCgcHx8Hi8fOezvpQkBMFIuBwTQ9v69QtsY25iQlpmJklpaWVkVQki9iQISgl9+U5NvB0B5+Kf6bxIQhCMBHOV6ok3eq0sk5GVhZmpaRlZVUJ4eOhCUMXOZUEpYgyzBSEIRkLfNm3YHBBQYJv4jAw6NW2KuUpVRlaVII0aGdoCgRHwrM8WhCAYCVP69eOHPXuIvHMn1/FLUVFcCA/n3+vXiUpKIiIiAveJE+nx7rsGsrQYxNwockW1/7d37+FR1Xcex9/fyR0UFIi1gDbKBjSC9RKnanHFK+iKrKK1tenKmtbdsltv61Ktdu3T2n10qVqttajrpT5h3a0XdF1FlEeUiheKLroRAqssCopykwSJTBLmu3+cMxAhkACZOcmZz+t5eJjMOcz5HpLMd363709kd8S1taCEkCdGVVRw7cSJnHzddbQUFnLshg1UNzZySksL57szZssWDkgkeK+8nAX9+3c8C6kny9Q3EsmROLYWlBDyyJUTJnBbbS1l++7LaqDvAQewPpHAEwl+VV7OykMO+dL6BBHpXJxaC5GsQzCzC4GfAYcDSXfXDmx7qXHTJupeeom33n8fM+Mbw4dz8ckn07e0lE2bN/PI3Lm8vmQJ6XSa740ZwzeGD2dTKsXfTp3KdUVFvJJKUbt6NSOLi7mkX7+ob2fv1NVBTU3UUUgeSSaBtWdTX/gqr7zde9ctRNVCqAfOB+ZGdP1Yuee556j4/vd5ub6e40eMIFlZyTMLFnBwbS0/uuceDq6t5en58zmuspITDjuMPy5axJk33sij8+bxfmMj//b55xxTUsI3y8p4u6WFYR98wCfNzbh71Le227aWxVbBO4lAb28tRNJCcPfFgLonusGDs2dzy+OPM//WW6kcPHjr85eNG8ftTz7JNQ8+yNRJk7j6vPO2HvvB2LHUzZnDJXfcQXlpKbOGDNl6rLZfP1a2tTF8xQqmPvEEUyZOzOn9dIfxg99SwTuJTG9uLViUnwLN7CXgmq52GVVXVvqC227LblC9SKq1lYMvvZTZv/gFV9x555cGgt2d+vXr+dqAAby/di2DgEz6TQOfAUXAZqC83bGCRNBo3FJYyLq2No7Yf38KE19uSPbr358Xe/L3oa6Opz8+BgYPUdeRRGr+fNj37GcBGP316GoiDSsZ9qa7V3d2XtZaCGY2G+jo7q9396d243UuAy4DOLi8vJuii4cZr73GkRUVjKqo2FqOIuP55mauCxeYJYDfA+PCFtmj7vwY6At8aMbtBQVcXFDAoi1bqKqoAKC6sZGWpiZqzfhRu9fNHOvRttY3ijoQyXe9rbWQtYTg7qd30+vcC9wLQQuhO14zLt798ENGV3U81XJRSwvHl5byijtFwCJgXHjsfeBE4B1gHzMW7aSVuE9REYtaWro/cJE8M7LtROY/C/U8Sz0NkbYWdkXTTnuxosJCNnfwhj1/82bqmpq4p7GRd9ev5wugDngjfOMvIOgqgqBcxc4+Fbg7hb15nEeDy9KD9IZ1C5EkBDM7z8xWAicAz5jZrCji6O1OGTWKGa+//qXZQHUbN3LuqlWc0acP5QUFHDlgAAngTGAC8LA7xwIvAm3ABnfGJDr+MfgslWJMWVn2byQbkknGj/l8214JIj3E9jORXlze0GOSQ1SzjGYAM6K4dpyMrqqiuLCQR+bOZeWaNTy+Zg2XA/8KHLJhAy8A9evWUQB8BzgeqAWmAX8OzAEcSLa28gXQCixavhyAD9JpWoAJffvm+ra6TzIJLy2KOgqRHWTGFubPh9Shy1h/WAMfD2yIfPtObZDTi5kZD191FWfeeCObgL8BSoF/BlYSvNmngIOASQRdRSngEoJv/EagGBhG0FRMAAe4s96ddek0Xx88mOObmna4br/tBplFZM8EezoFG/LUb3iVx9Y1MHIkkQ08RzrtdHdp2mnH/mf5co67/HJagLFmlJrxdDrN2QTdRE8B84CTgJkELYEzyspYVlTEulQKb23lpNJSis14dfNmKouK+LS0lIaHH47uprpLZgrqlB9HHYlIpxauXUbJYQ0MGEi3thYin3YquTOqooKBiQSfptNcUlBAETAvneZ1YDbQBzgbyOyG4AQtg4+bmykwY9qgQZQmEmwBfj5wIFXFxT1/amlX1dTAv6jbSHqHzPadUbUWNMsoRgYDxVu2cFFbG2uARoJv8DqCWUZPAJk5STO/+AJ3py2dpmb1aiavWcMxxcVU9ca9ELpCZbGlFxnZdiKphsOor4cXl+duwFkJIUYmAhPdMaCSoHuoBJgFvE3QMgCY1Lcv84YOpc2dr/bty/qKCoYUFDBqxQo+aWuLJvgsGj+lKthRTVNQpRc5atChHL72bNa8vx+PvZmbmUjqMurBTr366g73JejXvz9LVqyA1tatzzWn09wZPi4FVhNMNd0X+DbbWgYAv9+0iZmbNlEKLG9qYkRTE06wNqFi+XIGJhJQVLTL6/fo0hUiMTKy7UQWNiyjnuzPRFJC6MG2L0eRUd3YCK2tfNRu7+MDUynSwJHApwS1ioYALxOsUP6AIFEcBPwv8AlwLMH4wlUEpWf7ESSFFRUVJDdu3PX1e6OlSzPTOkR6lVyNLajLKGb+g6C2OEAzQYmKDwmmm84Abg+PZTqGTgP+O3xcSjAD6ZN0Oiex5lRNTbByua4u6khE9li2xxaUEGImRTBe0AasBVYRzC5y4BzgL8PzphNUPU2z4w9BXJuN48d8HnUIIntt+7GF7txzQQkhRoxg3OCM8PFXgL8gGFguBLYQjCkAPAi8R7BG4YTwuWag1IxBOyllEQtqJUhMZFoLr7y9odtaCzH+zY+fU1esoHr5clatW8eWdJrqVIrqVIpTwwJ3lQSDybcAAwlaCcsIxga+Er5GZh+EjQStiWUESaSFICGc17cvibgmhGRy245qIjHQ3a2FuPYOxEK//v2/NIC7qrWVFxIJCoqKgiqn4Srzse6kCD7xA0whGA9oIUgA+wBXADeEx8uBGuBqgqQxEMgUqFhUUkJ1Y+PW8hQdDSCrdIVIz5KZifQKDQwYuGGPZyIpIfRg20/trK6tpaqDN+OhjY0suP9+CiZM4LEpU/irX/+a5lQKCFYkNxEkgwKCTXE+Bv4x/LqkpIQNqRSHDR3KH2++mUH9+mX1nnqM+fM140hiZfuZSHuy50JM+wbyU9qdsUcfzaZHH6UMGBD+KQEeAO4EfhCe+0OCTwOHDx3KZWPHsvjuu/MmGWwti62FahJDezO2oIQQIwkzXluyBIASM05OJFhVUsIRwF8Dk4ERBAPOvykpYWAiwTHDhjFiyJDogo5CMhmsXBaJqe3HFrpKCSFGjj70UH46fToQjCHMSaf5ZLtqtncBR4W7oKXdeezVV/nW6NE5jlREciGzGU9XaQyhF9l+kLn98wC/uvRSTr3hBu6ZOZNEcTGtqRQVLS2UAQcSzCLaCAxwZ3BLC41mTKyuZuigQbm8jZ4js5uaxhIkxnbnx1sJoRfprH7QmFGj+PnFF/PDadM4rrKS+y66iN899xxLP/qItDufrlrFTy68kGRlJbc++SSFBQVMmzw5R9H3MDU1jK+r42lOiToSkR5DCSFmbrjoIkZXVXHNAw8w/qabSIfVT/uWllJSVMStTz7JMcOGMfmss7jopJMoKszzH4GX5qiFIBLSjmmS3+rqeJrxQa0jkZg691zr0o5pGlSW/DZ8eNQRiPQYSggiIgIoIUi+SyaD2kbaYlNECUFk/JSqqEMQ6RGUEEQyVBZb8pwSgghsK4ut+kaSx5QQREKqbyT5TglBREQAJQSRbYYPD1YuayxB8pQSgkhGMhnslSCSp5QQREQEUEIQ2dHHH6nbSPKSEoJIe+o2kjymhCDSkY8/ijoCkZyLJCGY2VQzazCzd8xshpntF0UcIh3K7Lms+kaSZ6JqIbwAjHT3I4GlwHURxSHSMe2PIHkokoTg7s+7e1v45evA0CjiEBGRbXrCGMKlwMydHTSzy8xsgZktWNPBBvMiWaVuI8kjWUsIZjbbzOo7+DOh3TnXA23A9J29jrvf6+7V7l5d3r9/tsIV2cH4KVXBWIIK3kmeyNoO6+5++q6Om9kk4BzgNO9NGztL/lm6NNhIRyTmopplNA6YApzr7s1RxCDSJTU1WqgmeSOqMYS7gH2BF8xsoZlNiygOkU5poZrki6x1Ge2Ku/9ZFNcVEZGd6wmzjER6PnUbSR5QQhDpTDK5bYtNkRhTQhDZHZqCKjGmhCDSRePHfB7sqCYSU0oIIl2ltQgSc0oIIiICKCGI7JatZbE1liAxpIQgsjtqaoKkIBJDSggiIgIoIYjsGc02khhSQhDZXZluI61clphRQhDZE8OHRx2BSLdTQhDZE8lkUMpCO6pJjFhv2pvGzNYAH0QdR2gQsDbqICKUz/eve89fvfX+v+bu5Z2d1KsSQk9iZgvcvTrqOKKSz/eve8/Pe4f437+6jEREBFBCEBGRkBLCnrs36gAils/3r3vPX7G+f40hiIgIoBaCiIiElBD2gplNNbMGM3vHzGaY2X5Rx5QrZnahmb1rZmkzi+2si+2Z2TgzW2Jm75nZtVHHkytm9oCZrTaz+qhjyTUzO8jM5pjZovBn/oqoY8oWJYS98wIw0t2PBJYC10UcTy7VA+cDc6MOJFfMrAD4LXAWUAV8x8yqoo0qZx4CxkUdRETagH9w9yrgeODv4vp9V0LYC+7+vLu3hV++DgyNMp5ccvfF7r4k6jhyLAm85+7L3L0F+HdgQsQx5YS7zwXWRx1HFNx9lbu/FT7eCCwGhkQbVXYoIXSfS4GZUQchWTUEWNHu65XE9I1BOmZmFcDRwBvRRpIdhVEH0NOZ2WzgwA4OXe/uT4XnXE/QrJyey9iyrSv3LpIvzGwf4HHgSndvijqebFBC6IS7n76r42Y2CTgHOM1jNoe3s3vPQx8BB7X7emj4nMScmRURJIPp7v5E1PFki7qM9oKZjQOmAOe6e3PU8UjW/QmoNLNDzKwY+DbwnxHHJFlmZgbcDyx299uijieblBD2zl3AvsALZrbQzKZFHVCumNl5ZrYSOAF4xsxmRR1TtoUTCP4emEUwsPgHd3832qhyw8weAV4DRpjZSjOrjTqmHPom8D3g1PD3fKGZnR11UNmglcoiIgKohSAiIiElBBERAZQQREQkpIQgIiKAEoKIiISUECTvmdkkMxvchfMeMrMLuvp8N8T1k3aPK/Kx0qjklhKCCEwCOk0IEfhJ56eIdB8lBImV8JN0g5lNN7PFZvaYmfUJjx1rZi+b2ZtmNsvMvhp+sq8GpocLjsrM7J/M7E9mVm9m94YrVbt6/R2uET7/kpndYmbzzWypmZ0UPt/HzP4Q1tqfYWZvmFm1md0MlIUxZWpkFZjZfWFN/ufNrKx7//ck3ykhSByNAO5298OBJmByWIvmN8AF7n4s8ADwS3d/DFgAfNfdj3L3L4C73P04dx8JlBHUqurUzq7R7pRCd08CVwI3hs9NBj4La+3/FDgWwN2vBb4IY/pueG4l8Ft3PwLYAEzc/f8akZ1TcTuJoxXuPi98XAdcDjwHjCQoMwJQAKzayb8/xcymAH2AAcC7wNNduO6ITq6RKYr2JlARPh4N3AHg7vVm9s4uXv//3H1hB68h0i2UECSOtq/H4oAB77r7Cbv6h2ZWCtwNVLv7CjP7GVDaxet2do1U+PcW9ux3L9Xu8RaC1otIt1GXkcTRwWaWeVO+GHgFWAKUZ543syIzOyI8ZyNBkULY9ua/Nqx/vzuzh3Z1jZ2ZB3wrPL8KGNXuWGvYDSWSE0oIEkdLCPa9XQzsD/wu3PLyAuAWM3sbWAicGJ7/EDDNzBYSfAq/j2DP6FkEJa+7pJNr7MzdBElkEXATQfdUY3jsXuCddoPKIlmlaqcSK+EWh/8VDgj3eGZWABS5+2YzGwbMBkaEyUUkpzSGIBKtPsCcsGvIgMlKBhIVtRBERATQGIKIiISUEEREBFBCEBGRkBKCiIgASggiIhJSQhAREQD+H7HYMwhhQlE6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_combined_std = np.vstack((X_train_std, X_test_std))\n",
    "y_combined = np.hstack((y_train, y_test))\n",
    "plot_decision_region(X_combined_std, y_combined, classifier=lr, test_idx=range(105, 150))\n",
    "plt.xlabel('petal length')\n",
    "plt.ylabel('petal width')\n",
    "plt.legend(loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is c? And what is regularization?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.89820289,  1.44587881],\n",
       "       [-1.16537974, -1.04507821],\n",
       "       [-1.33269725, -1.17618121]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_std[:3, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.17983737e-08, 1.44886616e-01, 8.55113353e-01],\n",
       "       [8.33962295e-01, 1.66037705e-01, 4.55557009e-12],\n",
       "       [8.48762934e-01, 1.51237066e-01, 4.63166788e-13]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict_proba(X_test_std[:3, :]) #every row sums up to 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1.])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict_proba(X_test_std[:3, :]).sum(axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 0])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict_proba(X_test_std[:3, :]).argmax(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 0])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#the above is the samething as\n",
    "lr.predict(X_test_std[:3, :])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "scikit-learn expects a two-dimentional arrays as data input, so you have to use numpy's **reshape** function to convert your input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.89820289, 1.44587881])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_std[0, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.89820289, 1.44587881]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_std[0, :].reshape(1, -1) # -1 in reshape is just a placeholder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict(X_test_std[0,:].reshape(1,-1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Overfitting and Regularization\n",
    "\n",
    "Regularization is used to model the complexity of a model. The regularization parameter in the model introduced penalty for extreme weights."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cost function for logistic regression can be regularized by shrinking the weights in the model. (lamda increase, regularization strenth increase, more penalty)\n",
    "\n",
    "The parameter c is the inverse of lamda. So decreasing the value of c will increase the regularization strength."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEOCAYAAACJlmBtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd8VGW+x/HPLz2EThJAihSlRogSQAWsLKJiWWXXggVxRUVdXe+9uru6tqu7a9nGKuvqFdAVFS6uYu8oqFw1QVQggKKACBJ6S0ImM8/94wwllDBAJmfK9/16zWtyznNmztfHYX5z2nPMOYeIiEiK3wFERCQ2qCCIiAiggiAiImEqCCIiAqggiIhImAqCiIgAKggiIhKmgiAiIoAKgoiIhKkgiIgIAGl+BzgQubm5rkOHDn7HEBGJKyUlJWucc3n7Wy6uCkKHDh0oLi72O4aISFwxs6WRLKddRiIiAqggiIhImAqCiIgAcXYMYW8CgQDLly+nsrLS7yhJJysri7Zt25Kenu53FBGpA3FfEJYvX06jRo3o0KEDZuZ3nKThnGPt2rUsX76cjh07+h1HROpA3O8yqqyspEWLFioG9czMaNGihbbMRBJI3G8hACoGPlG/S6IJBENUBoJUBIJUVoWoCAQJhhwOx/a7DTvHjmmHt7XswvOpMX9n266vY9f2vbwPu87fpe2Yw5uR2zAzqv/9CVEQ4s3EiRMZMmQIhx12WK3LjRw5kmHDhjF8+PCI5h+qhg0bsmXLlj3mv/jii3Tp0oUePXrU6fpEIuGcoyoY2vEFvf0L2/vSDlJZHaQi3FYRCLItEKSiapdlAuEv+argjtfXeI9AiMpwW3Uodu8xP/GKvpzUNT+q61BB8MHEiRMpKCjYb0GIFS+++CLDhg1TQZCo+6ZsC2/O+5G35v3I8vUVO77AD+Z7OjXFaJCeSlZGKlnpKWSnp5KdnkpWeirNcjJonZZKdoY37c0PLxOelxWel5ZigGEGhrdl7D2HH3gNe7Tt8hogvMxu77Pjb/a9jvD8w1s0OOT+3R8VhEO0ZMkShg4dSp8+fZg9ezY9e/bkqaeeokGDBpSUlHDzzTezZcsWcnNzmThxIh999BHFxcWMGDGC7OxsZs2axYMPPsjLL79MRUUFxx9/PP/85z8j3h2zt3W0bt2ak046if79+zN9+nQ2bNjAE088waBBgygvL2fkyJHMnTuXrl27smLFCh555BGKiooAuO2223jllVfIzs5m2rRpLF68mJdeeokPPviAe++9l+eff57OnTtHs0sliTjnKF25mTfmruT1uT/ydZm3hVrYrimnFbSiwW5f0N4XdgpZad4X/a5f8tnpqWRlpOyYTk+N+0Ok9S6hCsLdL89j/opNdfqePQ5rzJ1n9ax1mYULF/LEE08wYMAARo0axbhx47jxxhu54YYbmDZtGnl5eUyePJnbbruN8ePH8/DDD/PQQw/t+BK+/vrrueOOOwC49NJLeeWVVzjrrLP2my0QCOxzHQDV1dV8+umnvPbaa9x999288847jBs3jmbNmjF//nzmzp1LYWHhjvfbunUrxx57LPfddx+33HILjz/+OLfffjtnn312VHZRSXJyzvHF8o28Pnclb879kSVry0kx6NuhOXed1YPTClrRukm23zGTUkIVBL+0a9eOAQMGAHDJJZcwduxYhg4dyty5c/nJT34CQDAYpHXr1nt9/fTp03nggQcoLy9n3bp19OzZM6KCsHDhwlrXcd555wHQp08flixZAsCHH37IjTfeCEBBQQG9evXasXxGRgbDhg3b8Zq33377QLpBZJ+CIUfJ0vU7isCKjZWkpRjHdW7B6BM6M6Rny6gfMJX9S6iCsL9f8tGy++4dM8M5R8+ePZk1a1atr62srGTMmDEUFxfTrl077rrrrohP5dzfOjIzvX9gqampVFdX7/f90tPTd/y3RPoakX0JBEN88u06rwjMW8WaLdvISEvhhCNzuXlIVwZ3z6dpgwy/Y8ouEqog+GXZsmXMmjWL4447jmeeeYaBAwfStWtXVq9evWN+IBBg0aJF9OzZk0aNGrF582aAHV/+ubm5bNmyhalTp0a8a6a2dezLgAEDmDJlCieffDLz58/nq6++2u96ds0rUptt1UE+/HoNb8z9kbdLV7GhPEB2eiond8tjaEFrTumWT8NMfe3EKv2fqQNdu3blkUceYdSoUfTo0YNrr72WjIwMpk6dyi9/+Us2btxIdXU1N910Ez179mTkyJFcc801Ow4qX3XVVRQUFNCqVSv69u0b8XprW8e+jBkzhssvv5wePXrQrVs3evbsSZMmTWpdz4UXXshVV13F2LFjmTp1qg4qSw0VVUHeX1jG63N/5L0FZWzZVk2jrDQGd2/JaT1bcWKXPLIzUv2OKREw52L3vNvdFRUVud3vh1BaWkr37t19SuSdZTRs2DDmzp3rW4YDEQwGCQQCZGVlsXjxYgYPHszChQvJyDi4TXe/+1/8sbkywHsLynj9qx95f1EZlYEQzRqkM6RHK4Ye1YoBnXPJSNNZPrHCzEqcc0X7W05bCEmmvLyck08+mUAggHOOcePGHXQxkOSyfmsVb5eu4o25P/Lh12uoCobIb5TJz/q04/SCVvTr2Jw0neoZ11QQDlGHDh3iZusAvOMBuuucRKpscyVvzfOKwKxv1xIMOdo0zebS4w7n9IJWHNO+GSkpGsIkUaggiEgNKzZU8MbcH3lj7o98tnQdzkGn3ByuPqETQwtacVSbJhrHKkGpIIgI4J3GfOdL83hqlnf73W6tGnHjqUdyekFrurRsqCKQBFQQRASAZz/9nqdmLeWifu25alBHOuU19DuS1DMVBBHhi+83cNdL8zixSx73nVug4wJJSqcE+GDixImsWLFiv8uNHDmSqVOnHvD7P/roozz11FN7zF+yZAkFBQUAzJkzh9dee21H21133cVDDz10wOuS+Ld+axVjJs0mr1Emf72gUMUgiflWEMysnZlNN7P5ZjbPzG70K0t9i7QgHKxrrrmGyy67rNZldi8IkpyCIceNk+ewevM2xo04hmY5OgU5mfm5hVAN/IdzrgdwLHCdmcXdgPtLliyhW7dujBgxgu7duzN8+HDKy8sBb2jqE088kT59+nDaaaexcuVKpk6dumP468LCQioqKrjnnnvo27cvBQUFjB49mtouFiwrK6NPnz4AfPHFF5gZy5YtA6Bz586Ul5fX+LVfUlJC79696d27N4888ggAVVVV3HHHHUyePJnCwkImT54MwPz58znppJPo1KkTY8eOjVqfSez4+3tfM2PRau48uwe92zX1O474zLeC4Jxb6ZybHf57M1AKtDnkN55w5p6PTx/32qrK997++SSvfevaPdsisHDhQsaMGUNpaSmNGzdm3LhxO4amnjp1KiUlJYwaNYrbbruN4cOHU1RUxKRJk5gzZw7Z2dlcf/31fPbZZ8ydO5eKigpeeeWVfa4rPz+fyspKNm3axMyZMykqKmLmzJksXbqU/Px8GjSoeRONK664gr///e988cUXO+ZlZGRwzz33cMEFFzBnzhwuuOACABYsWMCbb77Jp59+yt13300gEDiAjpd48/7CMv727tecf0xbLu7X3u84EgNi4hiCmXUAjgY+8TfJwdl9+OsPP/ywxtDUhYWF3HvvvSxfvnyvr58+fTr9+/fnqKOO4r333mPevHm1ru/444/no48+YsaMGfz2t79lxowZzJw5k0GDBtVYbsOGDWzYsIETTjgB8O61UJszzzyTzMxMcnNzyc/PZ9WqVZF2gcSZ5evLuWnyHLq2bMS95xbolFIBYuAsIzNrCDwP3OSc2+PuNmY2GhgN0L59BL9irnh1320ZDWpvz2lRe/s+1Pfw1yeccMKOrYJzzjmH+++/HzPjzDMj26LZl+3DZYOGv05k26qDjJk0m2DQ8eglfTTwnOzg6xaCmaXjFYNJzrl/720Z59xjzrki51xRXl5e/QaM0Pbhr4G9Dn8N3t3Ntv/y39/w1/szaNAgnn76aY488khSUlJo3rw5r732GgMHDqyxXNOmTWnatCkffvghAJMmTdrRpiGtk9c9L8/ny+Ub+dPPe9MhN8fvOBJD/DzLyIAngFLn3J/9ylEXtg9/3b17d9avX19j+Otbb72V3r17U1hYyMcffwywY/jrwsJCMjMzdwx/fdppp0U0/HWHDh1wzu3YFTRw4ECaNm1Ks2bN9lh2woQJXHfddRQWFtY4WL39fgi7HlSWxPd8yXImfbKMa07szJCerfyOIzHGt+GvzWwgMBP4CgiFZ//WObfPcyE1/HXs8bv/JXKlKzfx03EfUdiuKU9f2V8jkyaRmB/+2jn3IaAjWSL1YFNlgGufLqFxVjp/v+gYFQPZK98PKse7eBv+WpKPc47/nPIFy9dX8NzoY8lrpJvZy97pZ4JIgntsxre8NX8VvzmjO0UdmvsdR2JYQhSEeLoNaCJRv8e+WYvXcv8bCzjzqNaMGtDB7zgS4+K+IGRlZbF27Vp9OdUz5xxr164lKyvL7yiyD6s2VXLDs5/TITeH+4f30sVnsl9xfwyhbdu2LF++nNWrV/sdJelkZWXRtm1bv2PIXgSCIa6bNJvyqmqevao/DTPj/p+61IO4/5Skp6fTsWNHv2OIxJQ/vr6A4qXr+duFhRzZspHfcSROxP0uIxGp6dUvV/LEh98x8vgOnFN46ONFSvJQQRBJIN+UbeGWqV9wTPum/PYMXTAoB0YFQSRBbN1WzbVPl5CVnsojI44hI03/vOXAxP0xBBHxzvr6zb+/YvHqLfzryv60bpLtdySJQ/oJIZIAnpq1lJe+WMF/DOnKgCNy/Y4jcUoFQSTOlSxdz72vzmdw93yuPbGz33EkjqkgiMSxtVu2cd2k2bRuks2fflZISoouPpODp2MIInEqGHL88rnPWV9exb/HHE+TBul+R5I4p4IgEqf+8vYiPvpmLQ8M70XPw5r4HUcSgHYZicShd0tX8fD0b7iwbzt+XtTO7ziSIFQQROLMsrXl/GryHAraNOaus3v6HUcSiAqCSBypDAS5dlIJAP8Y0Yes9FSfE0ki0TEEkThy57R5zFuxifEji2jXvIHfcSTBaAtBJE5M/mwZk4u/54ZTjuCUbi39jiMJSAVBJA7M/WEjv5s2j4FH5HLT4C5+x5EEpYIgEuM2lFdxzdMl5OZk8LcLC0nVxWcSJTqGIBLDQiHHzVO+YNWmSqZcfRwtGmb6HUkSmLYQRGLYuPe/4b0FZdwxrAdHt2/mdxxJcCoIIjFq5ter+dPbizi38DAuOfZwv+NIElBBEIlBKzZUcONzczgyvyG/P+8ozHTcQKJPBUEkxlRVhxgzaTZV1SH+cUkfGmToUJ/UD33SRGLMfa/OZ873G/jHiGPonNfQ7ziSRLSFIBJDps35gSdnLeWqQR05/ajWfseRJKOCIBIjFq3azK+f/4p+HZpzy9BufseRJLTfgmBmHSOZJyIHb3NlgGv+VUJOZhoPX3w06an6rSb1L5JP3fN7mTe1roOIJKuq6hDXP/M5S9eV88jFR5PfOMvvSJKk9nlQ2cy6AT2BJmZ23i5NjQF9YkXqQCjk+K+pX/DBotX88byj6N+phd+RJInVdpZRV2AY0BQ4a5f5m4GrohlKJBk457jnlflMm7OCW4Z25cJ+7f2OJElunwXBOTcNmGZmxznnZkVj5WY2Hq/olDnnCqKxDpFY9cj0b5j48RKuHNiRa0/s7HcckYiuQ/jGzH4LdNh1eefcqDpY/0TgYeCpOngvkbjxzCfLeOitRZx3dBtuO6O7rkSWmBBJQZgGzATeAYJ1uXLn3Awz61CX7ykS617/aiW3v/gVJ3fN4/7hvUjRcNYSIyIpCA2cc7dGPYlIEvh48RpufG4OR7dvxrgRfXR6qcSUSD6Nr5jZGVFPsg9mNtrMis2sePXq1X7FEDlkc3/YyOinSuiQ24AnLi8iOyPV70giNURSEG7EKwqVZrbJzDab2aZoB9vOOfeYc67IOVeUl5dXX6sVqVPfrdnK5eM/pUl2Ok+N6k/TBhl+RxLZw353GTnnGtVHEJFEtWpTJZc+8QkO+NeV/WjVRJfxSGyKZOgKM7NLzOx34el2ZtavLlZuZs8Cs4CuZrbczK6si/cViRUbywNc9sSnrN9axcQr+tJJo5dKDIvkoPI4IAScAvw3sAV4BOh7qCt3zl10qO8hEqsqqoJc+eRnfLdmKxOu6Euvtk39jiRSq0gKQn/n3DFm9jmAc269mWkHqEgtAsEQ1z0zm5Jl63nk4mMYcESu35FE9iuSg8oBM0sFHICZ5eFtMYjIXoRCjluf/5L3FpTx3+cUcIbuayBxIpKCMBZ4Acg3s/uAD4HfRzWVSJxyzvGH10v59+wfuPknXbjk2MP9jiQSsUjOMppkZiXAqYAB5zrnSqOeTCQO/XPGtzw+8zsuP+5wbjjlCL/jiByQ2oa/buyc22RmzYEy4Nld2po759bVR0CReDHls+/54+sLOLv3Ydx5Vk+NTyRxp7YthGfwRiItIXz8IMzC052imEskrrw9fxW//veXDDoyl4d+1lvjE0lcqm3462HhZ90uU6QWn3y7luuemc1RbZvy6CV9yEjT+EQSnyK5MO2nZtZkl+mmZnZudGOJxIf5KzbxiyeLadcsmwkj+5KTGcmZ3CKxKZKfMnc65zZun3DObQDujF4kkfiwbG05l43/lIZZafzryv40z9HlORLfIikIe1tGP4MkqZVtruSSJz6hOhTiX1f247Cm2X5HEjlkkRSEYjP7s5l1Dj/+jHegWSQpbaoMcPn4z1i9eRsTRvbliHyN/yiJIZKCcANQBUwOP7YB10UzlEisqgwE+cWTxXxTtplHL+3D0e2b+R1JpM5EcmHaVuDX9ZBFJKZVB0P88tnP+WzJOv56QSEndtH9OSSx1HZh2l+dczeZ2cvUvA4BAOfc2VFNJhJDnHPc9sJc3pq/irvO6sE5hW38jiRS52rbQngq/PxQfQQRiWUPvLmQycXf88tTjmDkAF2aI4mptoLwIN74RWc4526tpzwiMed/Zn7LP95fzMX92/Orn3TxO45I1NRWEFqb2fHA2Wb2HN6QFTs452ZHNZlIDPj37OXc+2opZxzViv8+p0DjE0lCq60g3AH8DmgL/ImaBcHh3UFNJGG9t2AV/zX1S47v3IK/XFBIqsYnkgRXW0FY6Zw73czucM7dU2+JRGJA8ZJ1jJk0mx6tG/PYZUVkpqX6HUkk6mq7DmFs+FnjFklSWfjjZkZN/IzDmmQz4Yq+NNT4RJIkavukB8zsMaCNmY3dvdE598voxRLxx/fryrls/CdkZ6Ty5Kh+5DbM9DuSSL2prSAMAwYDp6GhKiQJrNmyjcvGf0pFVZD/veZ42jVv4HckkXpV2/0Q1gDPmVmpc+6LeswkUu82VwYYOeFTVm6s4Okr+9O1lcYnkuQTyVhGFWb2rpnNBTCzXmZ2e5RzidSbbdVBrv5XCaUrNzNuxDEUdWjudyQRX0RSEB4HfgMEAJxzXwIXRjOUSH3Zsq2aX02ew8eL1/Lg8F6c0q2l35FEfBPJ6RMNnHOf7nZBTnWU8ohE3ffrynlvQRnvlK7ik2/XURUMcfuZ3TnvmLZ+RxPxVSQFYY2ZdSY8wJ2ZDQdWRjWVSB0Khhxzvl/PO6VlvFdaxsJVmwHolJfDyAEdGNKjpXYTiRBZQbgOeAzoZmY/AN8BI6KaSuQQba4MMGPRGt5dsIr3F65m3dYq0lKMfh2bc3tRd07t3pKOuTl+xxSJKZHcD+FbYLCZ5QApzrnN0Y8lcuCWrS3nndJVvLegjE++W0sg6GjaIJ2Tu+ZzSrd8TuiSR5PsdL9jisSs/RYEM2sC3AmcEJ7+ALjHObcxytlEalUdDDF72QbeXbCKd0vL+KZsCwBH5Ddk1MCODO7ekqPbNSUtNZJzJ0Qkkl1G44G5wM/D05cCE4DzohVKZF82VQb4YOFq3ltQxvSFZWwoD5CeavTv2IKL+7Xn1O75HN5Cu4JEDkYkBaGzc+78XabvNrM50Qoksrvv1mzl3VJvK+CzJeuoDjma52RwSrd8BndvyaAjc2mUpV1BIocqkoJQYWYDnXMfApjZAKAiurEkmVUHQxQvXe8VgQVlfLt6KwBdWjbkqhM6Mbh7PoXtmmk4apE6FklBuBZ4MnwsAWA9MDJqiSQpbSwP8P6iMt4tLeP9hWVsqqwmPdU4tlMLLjv2cE7t3lJjC4lEWSRnGc0BeptZ4/D0pqinkoQWCjk2VQZYubGSmV+v5t3SMoqXricYcrTIyWBIz1YM7p7PwCPzNPS0SD2K5Cyj3wMPOOc2hKebAf/hnDvk8YzMbCjwNyAV+B/n3B8P9T2lfgVDjo0VAdaXV7F+axXrywOs31rFuvKqvc7bUB5gQ3kVIbfzPbq1asQ1J3bi1O4tKWzblBTtChLxRSQ/v053zv12+4Rzbr2ZnQEcUkEws1TgEeAnwHLgMzN7yTk3/1DeVw5eMOTYUB7+At/xZV7Fuq3el/i6rXu2bagIkOm20YwtZFkVWXiPTAswz7rQICeHwsyVDEn5hiZZQRo3DNI4rZqGqdUs63ktRx/RlrZVS2D9YrDNsDEPcvIhQ7uHROpbJAUh1cwynXPbAMwsG6iLu4b0A74JX/iGmT0HnAPUeUFYtamSLSsWYqFt3ozwr9NQeg6BRu1wDjLWf42FqnHs/OkaymhEVUNvfJvMdQsxF6zxvsHMJlTltAEga+18wNV4fTCrOVUNWoMLkb12vtfqHM6FCAVDbMvOozK7NaHqbeSs+RIXChEKhXAuiHOOLdlt2dKgDSmBcpqt/oxQKAQu5L0+5Fjf8Ag2ZrUhtWoTh62d5b3ehSAUwuH4oeFRrM9sQ9a2NXRc9xHOhSC8/uqqSv4vtYhFgVwab1nMkPLXsGAlmQS8L3QC/Ln6Aha49pyU8jm/S59EAwuQZQEyqSLDVfFYl0epzO3Fsetf4vjSe/fod3fdZ1heF/j4YXjrrzUbU9Lpfd5/QeMG8M4U+PAvNdszGsJ/LoKMHJjzLHz/CTTMh5y88HM+tD8WdNN7kToTSUGYBLxrZhPC01cAT9bButsA3+8yvRzoXwfvu4eH3/uGS2dfRJeUH2rM/yDYi8sDvwbgo8wbaGNra7S/GuzHdYGbAPgi8xc0sfIa7VOqT+SW6qsB+DrzUtKtZsGYUH0ad1dfTiZVLMwauUeuv1efy5+qf04LNlKSde0e7fcHLuQfwbNpb6uYkfmrPdp/FxjJv4JD6G5LeT3zN3u031x1DS+4E+iXspDr0u/bo31ezm/IbnYSfZpt46fbZhJMz8SlZkJ6FqRl88gJ3cns0I/c9c3JLP4aS8+GtCwIP4/pexw0aQurz4EjW3ptO9ozsSZeseToS6DHOTteR1oWpO7y0TvuBq99y2rYWgZbyqB8LaSHtxLWLITSl6B8HTuqeWZj+E344/PiGPhuxi7FIg+ad4JBN3vtZQu8wpGTB9nNVERE9sGcc/tfyNvXPzg8+bZz7s1DXrE3SN5Q59wvwtOXAv2dc9fvttxoYDRA+/bt+yxduvSA1zX3h41UzH+TtOqtu7wxVGXmsj6vCIDclTNI3b4FEbYtuyUbW/TGgNwV00lxNQd5rcxpw+ZmPTEg74d3gJp9WdmwHVuadsNckNwV08PrTcFSUkixFLY1Ppyqpp1JcwEar/rUm59ipFgqlpJCsMnhuMaHkRLcRvba+aSkpGKpKZgZqSmpuMaHYTl5pAa3kbZxCZaSQmpqKmYppKSkkNooH8tqDIFK2Lra+yK0FO8/Pi0TMhtBahydvx+shvI1XsHYthk6DPDmF0/wtiC2lIULymrIyYVrP/Lax58Oyz72/k5J9wrD4cfB8PHevDnPeAUpQxe0SWIysxLnXNF+l4ukIESDmR0H3OWcOy08/RsA59wf9vWaoqIiV1xcXE8JJa45t3NL4PvPYMPSmgWjUSs49Xfecn/p6W1RXDxFxy4kIUVaEPw8p+8z4Egz6wj8gHfTnYt9zCOJZNfdQu36eo99LTf4Lnjhanjm5yoKktR8G/XLOVcNXA+8CZQCU5xz8/zKI0ms18/h3Edh6UdeUagq3/9rRBLQfguCmd0YybyD4Zx7zTnXxTnX2Tm351FPkfrS+4KdRaFkwv6XF0lAkWwhXL6XeSPrOIeI/3pfAFe8Af33PONLJBns8xiCmV2Et0+/o5m9tEtTI2BdtIOJ+KJ9+Mzn9Uvhg/vhjId0TEGSRm0HlT/Gu3dyLvCnXeZvBr6MZigR362Y7Z2OuukHuOg57xoKkQS3z4LgnFsKLAWOq784IjGi50+96zdevBaevVBFQZJCJAeVzzOzr81so5ltMrPNZqYRTyXxFV4E546Dbz+AZy+CgG4DIoktkusQHgDOcs6VRjuMSMwpvNi7eK34Cajepq0ESWiRFIRVKgaS1I4eAb0u8MZfClQCToVBElJtZxmdF/6z2MwmAy8COwb7cc79O8rZRGJHapq3pTD5EnAhuPAZbxBAkQRS2zGEs8KPxkA5MGSXecOiH00kxph5g+Atfg+euzi8tSCSOGo7y+iK+gwiEheOudR7fukGryhoS0ESSCS30By7l9kbgWLn3LS6jyQS43YUheu9wnD+4/7mEakjkRxUzgK6Af8bnj4f+A7obWYnO+duilY4kZh1zKWQkgatjvI7iUidiaQg9AIGOOfdP9LM/gHMBAYCX0Uxm0hsK7zIe3YOvpwSviucdh9J/IpkcLtmQMNdpnOA5uECsW3vLxFJIivnwAujYcql3rUKInEqkoLwADDHzCaY2UTgc+BBM8sB3olmOJG4cNjRcNbf4Ou3vNNSVRQkTu13l5Fz7gkzew3oF571W+fcivDf/xW1ZCLxpM9Ib9fRKzfB5Evhgn95960WiSP73EIws27h52OA1sD34Uer8DwR2VXRFTDsr/DNO7Bslt9pRA5YbVsINwOjqTn09XYOOCUqiUTiWdEV0OlEaN7J7yQiB6y2C9NGh59Prr84IglgezFY+AbMmQTn/492H0lciGT46wZmdruZPRaePtLMNHSFyP5sXgGlL8GUy3WgWeJCJGcZTQCqgOPD0z8A90YtkUiiKBo5wO0XAAAMOUlEQVQFZ/4JFr0eLgpVficSqVUkBaGzc+4BIADgnCsHLKqpRBJF319492Ve9Dr87+UQDPidSGSfIrlSucrMsvEOJGNmndEFaSKR63eV97x6oTfchUiMiuTTeRfwBtDOzCYBA4CRUcwkkni2FwWAjcshJx/SMvzLI7IXkVyY9paZlQDH4u0qutE5tybqyUQS0bbN8MQQ7+rm4RNUFCSmRHKW0dPAecBi59wrKgYihyCzEQy4CRa8AlOv0IFmiSmRHFR+Au9K5b+b2bdm9ryZ3RjlXCKJq/9oOP3BnUVBB5olRuy3IDjnpgP3Ab8DHgeKgGujnEsksfUfDac/4BWFDx7wO40IENkd097FG/J6Ft59EPo658qiHUwk4fW/GnJy4Yif+J1EBIhsl9GXeBemFeDdLKcgfBqqiByqgvMhqzFUbYUZD0Io6HciSWKRnGX0KwAza4R3uukEoBWgwVlE6srC1+G9e72CcNKv/U4jSSqSXUbXA4OAPsASYDzeriMRqSsF53vDZr//R2hTBEcO9juRJKFILkzLAv4MlDjnqqOcRyQ5mcGZf4Yfv4Lnr4SrZ0Czw/1OJUkmkrOMHnLOfaJiIBJlGQ28O61tv/OaSD2L5KBynTOzn5nZPDMLmVmRHxlEYlLzTnDh0949mkXqmS8FAZiLd/XzDJ/WLxK7Op4ATdtDKOTtQhKpJ74UBOdcqXNuoR/rFokbMx+Cx0+FFXP8TiJJwq8tBBHZn6JRkJMHUy6F8nV+p5EkELWCYGbvmNncvTzOOcD3GW1mxWZWvHr16mjFFYk9Obnw8ydh00p44WpvF5JIFEWtIDjnBjvnCvbymHaA7/OYc67IOVeUl5cXrbgisaltEZz+R/j6Le9KZpEo0u2bRGJd0ZWw7jvoMNDvJJLg/Drt9Kdmthw4DnjVzN70I4dIXDCD0+6DDgO8ad1DQaLEr7OMXnDOtXXOZTrnWjrnTvMjh0jcmflnGH8aBCr9TiIJSGcZicSTvK6wYja8fovfSSQBqSCIxJNuZ8LAX8HsJ+Hzp/1OIwlGBUEk3px8u3c186v/ASu/8DuNJBAVBJF4k5oG54+HRq1gtS74l7qj005F4lHDPLjuU0jTfaqk7mgLQSRebS8G816Aj8b6m0USggqCSLxb+Dq8fYd3xzWRQ6CCIBLvhv0F8rvD87+ADcv8TiNxTAVBJN5l5MAFT0MoCFMu00VrctBUEEQSQYvOcO4/YMXnUPqy32kkTuksI5FE0X0YXD0DWvf2O4nEKW0hiCSS7cVg5Ze6/aYcMG0hiCSaYDX87+XeMYWrP4DsZn4nkjihLQSRRJOaBj/9J2xaAf/WndYkcioIIomoXT847ffw9Zsw809+p5E4oYIgkqj6XQVH/Qym3wfLPvE7jcQBHUMQSVRmcNbfoGUBtDnG7zQSB7SFIJLIMnJg4E2Qmg5b10L1Nr8TSQxTQRBJBpUb4Z8nwOu3+p1EYpgKgkgyyGoCR50PJRNgzjN+p5EYpYIgkixOuQM6DIJXfuVduCayGxUEkWSRmgbDx3sXqk25FCrW+51IYowKgkgyaZgPP3sScruCc36nkRij005Fkk37/jBiive3c97pqSJoC0EkeW1ZDRPPhMXT/U4iMUIFQSRZZTSA8nXw/JWwcbnfaSQGqCCIJKvtd1qrrvLutKaL1pKeCoJIMss9As4dBz+UwBu/8TuN+EwFQSTZ9Tgbjr8Bvp0OFRv8TiM+UkEQETj1Lhj9PmQ31f0TkphOOxUR76K11Cbe3xOGQoNc6HEOdB3qDXshSUEFQUR2ClbDYUfD/Jdg4auQkg6dT4Zjr4XOp/idTqJMu4xEZKfUNDj9fvjVPLjyHeh/NaxeABt/8No3r4LiCd41DJJwzMXR5etFRUWuuLjY7xgiycU5CAW9YvH50zDtOrAUOHyAt1up2zBo3NrvlFILMytxzhXtbzltIYhI7cy8YgBQOAKu+QgG/SdsXQ2v/Sf8pYd38x3wrmmQuOXLMQQzexA4C6gCFgNXOOd0vptIrDODVgXe45TbYPVC+P5TyGnhtU8eAeVrofvZ3umszTv5m1cOiF9bCG8DBc65XsAiQFfEiMSjvK5wzKU7pzueCC4E79wJY4+GRwfC55P8yycHxJeC4Jx7yzlXHZ78P6CtHzlEpI4df713PcONX8KQ+yAtG7as8tqqtsL0P8CqeRp6O0b5flDZzF4GJjvnnt5H+2hgNED79u37LF26tD7jicih2j7E9rcfwFPnAA6ad/YOSPc4B1r31hDcURbpQeWoFQQzewdotZem25xz08LL3AYUAee5CILoLCOROLelDBa8AvOnwXczwQXh2o+hZU/YthkyGqo4REGkBSFqB5Wdc4NrazezkcAw4NRIioGIJICG+VA0ynuUr4Nv3oX8Hl7b67/2xlPafkC6VS9Iz4aUVH8zJxG/zjIaCtwCnOicK/cjg4j4rEFz6PWzndNdhnj3eS4eD5/8w5t3+EC44lXv7/Gnw5YfIb2BVyjSs732k2712t+5C4KBXdobQH536HSi1754OqRl7mxLz4asppDVuN7+k2OdX0NXPAxkAm+bt3n4f865a3zKIiKxYPsxhW2b4Zt3YMMyaLTLBW9ti2DzSghUQKA8/Lx1Z/u8F70D2IEKILzToXDEzoIw6WcQCtRcZ9+r4MyHvELy+8NqFov0BtBnJPS7yitUz14c3p1lO3dr9RkJRw2HTSvhxWt3azfo+wtvPKj1S7wtoN1f3/8a6DgIVi+C6fft+frjxkCbPnXZy7XypSA4547wY70iEgcyG0HPn+45f8h/1/66G+d4z855N/sJlHtXVG+fN/JVqK6oWVBaHBluD8GxY3YrNhVelu1SUsNnR7nwiLDOe93211dt2dm+/bm6wmsPBmDT8nCd2qW9aovXHtgKZfP3fH09D0fu+1lGB0IHlUVEDpyGrhARkQOigiAiIoAKgoiIhKkgiIgIoIIgIiJhKggiIgKoIIiISJgKgoiIAHF2YZqZrQbiffzrXGCN3yFiiPpjJ/VFTeqPmg6lPw53zuXtb6G4KgiJwMyKI7liMFmoP3ZSX9Sk/qipPvpDu4xERARQQRARkTAVhPr3mN8BYoz6Yyf1RU3qj5qi3h86hiAiIoC2EEREJEwFQUREABUEEREJU0GIMWaWY2bFZjbM7yx+MrNzzexxM5tsZkP8zuOH8GfhyXA/jPA7j9/0magpGt8VKgh1xMzGm1mZmc3dbf5QM1toZt+Y2a8jeKtbgSnRSVk/6qIvnHMvOueuAq4BLohm3vp0gH1zHjA13A9n13vYenAg/ZGon4ntDuLfTZ1/V6gg1J2JwNBdZ5hZKvAIcDrQA7jIzHqY2VFm9spuj3wz+wkwHyir7/B1bCKH2Be7vPT28OsSxUQi7BugLfB9eLFgPWasTxOJvD+2S7TPxHYTifzfTVS+K9Lq8s2SmXNuhpl12G12P+Ab59y3AGb2HHCOc+4PwB6beWZ2EpCD9z++wsxec86Fopk7GuqoLwz4I/C6c252dBPXnwPpG2A5XlGYQ4L+eDuQ/jCzUhLwM7HdAX42GhKF7woVhOhqw85feOD9A++/r4Wdc7cBmNlIYE08FoNaHFBfADcAg4EmZnaEc+7RaIbz2b76ZizwsJmdCbzsRzCf7Ks/kukzsd1e+8I5dz3U/XeFCkIMcs5N9DuD35xzY/G+EJOWc24rcIXfOWKFPhN7quvvioTcDI0hPwDtdpluG56XjNQX+6a+qUn9sVO99oUKQnR9BhxpZh3NLAO4EHjJ50x+UV/sm/qmJvXHTvXaFyoIdcTMngVmAV3NbLmZXemcqwauB94ESoEpzrl5fuasD+qLfVPf1KT+2CkW+kKD24mICKAtBBERCVNBEBERQAVBRETCVBBERARQQRARkTAVBBERAVQQRA6JmbUys+fMbLGZlZjZa2bWxe9cIgdDYxmJHKTwiKwvAE865y4Mz+sNtAQW+ZlN5GCoIIgcvJOBwK6jbjrnvvAxj8gh0S4jkYNXAJT4HUKkrqggiIgIoIIgcijmAX38DiFSV1QQRA7ee0CmmY3ePsPMepnZIB8ziRw0FQSRg+S8oYJ/CgwOn3Y6D/gD8KO/yUQOjoa/FhERQFsIIiISpoIgIiKACoKIiISpIIiICKCCICIiYSoIIiICqCCIiEiYCoKIiADw/0jS4E0j/4c1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "weights, params = [], []\n",
    "\n",
    "for c in np.arange(-5, 5):\n",
    "    lr = LogisticRegression(C=10.**c, random_state=1)\n",
    "    lr.fit(X_train_std, y_train)\n",
    "    weights.append(lr.coef_[1]) #only for class 1, versicolor\n",
    "    params.append(10.**c)\n",
    "    \n",
    "weights = np.array(weights)\n",
    "plt.plot(params, weights[:,0], label = 'petal lenght')\n",
    "plt.plot(params, weights[:,1], linestyle = '--', label='petal width')\n",
    "plt.ylabel('weight coefficient')\n",
    "plt.xlabel('C')\n",
    "plt.legend(loc = 'upper left')\n",
    "plt.xscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "so 10 logistic regression models are fit and compared only for class 1(versicolor). small c means large lamda, greater regularization, more shrinkage of weights, smaller coefficients."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}