TensorFlow Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A TensorFlow tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import pyplot\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import log_loss\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# data source = https://www.kaggle.com/c/leaf-classification\n",
    "data = pd.read_csv('01-DATA/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(990, 194)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>species</th>\n",
       "      <th>margin1</th>\n",
       "      <th>margin2</th>\n",
       "      <th>margin3</th>\n",
       "      <th>margin4</th>\n",
       "      <th>margin5</th>\n",
       "      <th>margin6</th>\n",
       "      <th>margin7</th>\n",
       "      <th>margin8</th>\n",
       "      <th>...</th>\n",
       "      <th>texture55</th>\n",
       "      <th>texture56</th>\n",
       "      <th>texture57</th>\n",
       "      <th>texture58</th>\n",
       "      <th>texture59</th>\n",
       "      <th>texture60</th>\n",
       "      <th>texture61</th>\n",
       "      <th>texture62</th>\n",
       "      <th>texture63</th>\n",
       "      <th>texture64</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Acer_Opalus</td>\n",
       "      <td>0.007812</td>\n",
       "      <td>0.023438</td>\n",
       "      <td>0.023438</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>0.011719</td>\n",
       "      <td>0.009766</td>\n",
       "      <td>0.027344</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007812</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002930</td>\n",
       "      <td>0.002930</td>\n",
       "      <td>0.035156</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004883</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.025391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Pterocarya_Stenoptera</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031250</td>\n",
       "      <td>0.015625</td>\n",
       "      <td>0.025391</td>\n",
       "      <td>0.001953</td>\n",
       "      <td>0.019531</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.023438</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.039062</td>\n",
       "      <td>0.022461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Quercus_Hartwissiana</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.009766</td>\n",
       "      <td>0.019531</td>\n",
       "      <td>0.007812</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.068359</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.154300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.007812</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.020508</td>\n",
       "      <td>0.002930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>Tilia_Tomentosa</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>0.023438</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.021484</td>\n",
       "      <td>0.019531</td>\n",
       "      <td>0.023438</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.020508</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.017578</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.047852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>Quercus_Variabilis</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.003906</td>\n",
       "      <td>0.048828</td>\n",
       "      <td>0.009766</td>\n",
       "      <td>0.013672</td>\n",
       "      <td>0.015625</td>\n",
       "      <td>0.005859</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.096680</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021484</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.031250</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 194 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                species   margin1   margin2   margin3   margin4  \\\n",
       "0   1            Acer_Opalus  0.007812  0.023438  0.023438  0.003906   \n",
       "1   2  Pterocarya_Stenoptera  0.005859  0.000000  0.031250  0.015625   \n",
       "2   3   Quercus_Hartwissiana  0.005859  0.009766  0.019531  0.007812   \n",
       "3   5        Tilia_Tomentosa  0.000000  0.003906  0.023438  0.005859   \n",
       "4   6     Quercus_Variabilis  0.005859  0.003906  0.048828  0.009766   \n",
       "\n",
       "    margin5   margin6   margin7  margin8    ...      texture55  texture56  \\\n",
       "0  0.011719  0.009766  0.027344      0.0    ...       0.007812   0.000000   \n",
       "1  0.025391  0.001953  0.019531      0.0    ...       0.000977   0.000000   \n",
       "2  0.003906  0.005859  0.068359      0.0    ...       0.154300   0.000000   \n",
       "3  0.021484  0.019531  0.023438      0.0    ...       0.000000   0.000977   \n",
       "4  0.013672  0.015625  0.005859      0.0    ...       0.096680   0.000000   \n",
       "\n",
       "   texture57  texture58  texture59  texture60  texture61  texture62  \\\n",
       "0   0.002930   0.002930   0.035156        0.0        0.0   0.004883   \n",
       "1   0.000000   0.000977   0.023438        0.0        0.0   0.000977   \n",
       "2   0.005859   0.000977   0.007812        0.0        0.0   0.000000   \n",
       "3   0.000000   0.000000   0.020508        0.0        0.0   0.017578   \n",
       "4   0.021484   0.000000   0.000000        0.0        0.0   0.000000   \n",
       "\n",
       "   texture63  texture64  \n",
       "0   0.000000   0.025391  \n",
       "1   0.039062   0.022461  \n",
       "2   0.020508   0.002930  \n",
       "3   0.000000   0.047852  \n",
       "4   0.000000   0.031250  \n",
       "\n",
       "[5 rows x 194 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_classes = len(data['species'].unique())\n",
    "nb_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "le = preprocessing.LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y = le.fit_transform(data['species'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = data.drop('id', 1).drop('species', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=487)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.23374595993\n"
     ]
    }
   ],
   "source": [
    "clf = LogisticRegression()\n",
    "clf.fit(X_train, y_train)\n",
    "print log_loss(y_test, clf.predict_proba(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.80428024212\n"
     ]
    }
   ],
   "source": [
    "clf = RandomForestClassifier()\n",
    "clf.fit(X_train, y_train)\n",
    "preds = clf.predict(X_test)\n",
    "print log_loss(y_test, clf.predict_proba(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# TensorFlow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X_train_tf = tf.constant(np.array(X_train).astype(np.float32))\n",
    "    y_train_tf = tf.constant((np.arange(nb_classes) == y_train[:,None]).astype(np.float32))\n",
    "    X_test_tf = tf.constant(np.array(X_test).astype(np.float32))\n",
    "    y_test_tf = tf.constant((np.arange(nb_classes) == y_test[:,None]).astype(np.float32))\n",
    "    \n",
    "    weights = tf.Variable(\n",
    "        tf.truncated_normal([X.shape[1], nb_classes]))\n",
    "    biases = tf.Variable(tf.zeros([nb_classes]))\n",
    "\n",
    "    logits = tf.matmul(X_train_tf, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_train_tf))\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.8).minimize(loss)\n",
    "    \n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(X_test_tf, weights) + biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss at step 0: 4.624413\n",
      "Valid Loss: 4.631759\n",
      "Train Loss at step 1000: 4.137475\n",
      "Valid Loss: 4.195310\n",
      "Train Loss at step 2000: 3.732263\n",
      "Valid Loss: 3.823207\n",
      "Train Loss at step 3000: 3.384432\n",
      "Valid Loss: 3.503140\n",
      "Train Loss at step 4000: 3.086440\n",
      "Valid Loss: 3.228689\n",
      "Train Loss at step 5000: 2.829750\n",
      "Valid Loss: 2.991925\n",
      "Train Loss at step 6000: 2.606733\n",
      "Valid Loss: 2.785802\n",
      "Train Loss at step 7000: 2.411375\n",
      "Valid Loss: 2.604877\n",
      "Train Loss at step 8000: 2.239062\n",
      "Valid Loss: 2.445024\n",
      "Train Loss at step 9000: 2.086184\n",
      "Valid Loss: 2.303040\n",
      "Train Loss at step 10000: 1.949856\n",
      "Valid Loss: 2.176355\n"
     ]
    }
   ],
   "source": [
    "num_steps = 10001\n",
    "train_loss = []\n",
    "test_loss = []\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    for step in range(num_steps):\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction])\n",
    "        if (step % 1000 == 0):\n",
    "            train_loss.append(l)\n",
    "            test_loss.append(log_loss(y_test, test_prediction.eval()))\n",
    "            print('Train Loss at step %d: %f' % (step, l))\n",
    "            print('Valid Loss: %f'% log_loss(y_test, test_prediction.eval()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEPCAYAAABGP2P1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VEXbwOHfkwohQCgSpITQBAEloBRRIICAgGADjAUI\nvKIiiIrlswEWfO2iqK+KKMUCiKIioKBAKIL00Jv0Il0INSHJfH+czZ5sCiQkW5I893XtRc7M7NnZ\nybJPzsyZGTHGoJRSSqXx83YFlFJK+RYNDEoppVxoYFBKKeVCA4NSSikXGhiUUkq50MCglFLKhdsD\ng4jsEpE1IrJaRJZlU2aUiGwTkXgRiXJ3nZRSSmUvwAOvkQpEG2P+zSpTRDoBNY0xtUWkGfAp0NwD\n9VJKKZUFT3QlySVe5zZgAoAxZilQWkTCPVAvpZRSWfBEYDDA7yKyXET6Z5FfGdib7ni/I00ppZQX\neKIr6UZjzD8icgVWgNhkjFnkgddVSil1GdweGIwx/zj+PSIiPwJNgfSBYT9QNd1xFUeaCxHRRZ2U\nUuoyGGMkN+Xd2pUkIiEiEur4uQTQAVifodg0oLejTHPghDHmUFbnM8bowxiGDx/u9Tr4ykPbQttC\n2+Lij8vh7iuGcOBHx1/7AcA3xpjZIvIQYIwxo40xM0Wks4j8DZwB+rq5TgXerl27vF0Fn6FtYdO2\nsGlb5I1bA4MxZieQaV6CMeazDMeD3FkPpZRSOacznwug2NhYb1fBZ2hb2LQtbNoWeSOX2wflaSJi\nCkpdlVLKV4gIxpcGn5V7xMXFebsKPkPbwuaNtoiMjERE9OEDj8jIyHz7vXpiHoNSqpDavXv3Zd/5\novKXSK4uCi5+roLyS9WuJKV8j6ObwtvVUGT/u9CuJKWUUnmmgaEA0n51m7aFTdtC5RcNDEoplY0B\nAwbw2muvXdZz27Rpw5dffpnPNfIMHWNQSl02Xx5jqF69Ol988QVt27b1yuu3adOGXr160a9fP4+8\nno4xKKVUHqWkpHi7Cj5LA0MBpH3JNm0Lm7aFrXfv3uzZs4euXbtSqlQp3nnnHXbv3o2fnx9ffvkl\n1apVo127dgD07NmTK6+8kjJlyhAdHc3GjRud5+nbty/Dhg0DYP78+VStWpX33nuP8PBwKleuzLhx\n43JUH2MMI0aMIDIykooVKxIbG0tCQgIAiYmJ9OrVi/Lly1OmTBmaNWvGkSNHABg3bhw1a9akVKlS\n1KxZk4kTJ+ZjK2VPA4NSyi1E8veRGxMmTCAiIoLp06eTkJDAU0895cxbsGABmzdvZtasWQB07tyZ\n7du3c/jwYRo3bsx9992X7XkPHjzIqVOnOHDgAGPGjGHgwIGcPHnykvUZO3YsEyZMYP78+ezYsYNT\np07x6KOPAjB+/HgSEhLYv38/x48f59NPP6V48eKcPXuWxx57jFmzZpGQkMDixYuJisq09JxbaGAo\ngKKjo71dBZ+hbWHTtsgsY5+7iPDyyy9TvHhxgoODAWtdpZCQEAIDAxk2bBhr1qzh1KlTWZ4vKCiI\noUOH4u/vT6dOnQgNDWXLli2XrMe3337LkCFDqFatGiEhIbz++utMmjSJ1NRUAgMDOXbsGFu3bkVE\naNSoEaGhoQD4+/uzbt06zp8/T3h4OFdffXUeWyRnNDAopYqUKlWqOH9OTU3l2WefpVatWoSFhVG9\nenVEhKNHj2b53HLlyuHnZ39thoSEcPr06Uu+5oEDB6hWrZrzuFq1aly4cIFDhw7Rq1cvOnbsSExM\nDFWqVOHZZ58lJSWFkJAQJk+ezCeffMKVV15J165dcxSE8oMGhgJI+5Jt2hY2X2sLY/L3kVvZLRGR\nPv3bb7/ll19+Ye7cuZw4cYJdu3blaYOb7FSqVIndu3c7j3fv3k1gYCDh4eEEBAQwdOhQNmzYwOLF\ni/nll1+YMGECAO3bt2f27NkcPHiQOnXq0L9//3ytV3Y0MCilCqWKFSuyY8cOl7SMX/inTp0iODiY\nMmXKcObMGZ577rl8XXMozT333MPIkSPZtWsXp0+f5oUXXiAmJgY/Pz/i4uJYv349qamphIaGEhgY\niJ+fH4cPH2batGmcPXuWwMBAQkND8ff3z/e6ZUUDQwGkfck2bQubtoWrZ599lldffZWyZcvy3nvv\nAZmvInr37k1ERASVK1emQYMGtGjRIlevcbEgkj6vX79+9OrVi1atWlGzZk1CQkIYNWoUYA1od+/e\nndKlS1O/fn3n/IfU1FTee+89KleuTPny5VmwYAGffPJJrup3uQrUBLfzF84THBDs7aoopRx8eYJb\nUVNkJ7g1fv9mjpw54u1qeJ2v9SV7k7aFTdtC5ZcCFRg2nllEvfebsO7QOm9XRSmlCq0C1ZXES9bP\nwYQyJeZbutbp6tU6KVXUaVeS7yiyXUkkWpM+EjlNt4m38eait/RDqZRS+axABYaolUvg30jrQAzP\nzvk/+vwUS2Jyolfr5Wnal2zTtrBpW6j84pHAICJ+IrJKRKZlkddaRE448leJyIvZnefPHxvQdscy\n2N3SmfbV2glEj2vDodOH3FR7pZQqWjwyxiAiTwDXAaWMMd0y5LUGnsyYnsU5jDGGpCS4v08SU84+\nAo2/cOZXLRXBL/dMo2HFhu54C0qpLOgYg+8oUGMMIlIF6AyMuVixnJ4vKAgmfh3Ew5U/h9/eg1Tr\nLexN2MONX97IT5t/yluFlVKqiPNEV9JI4GngYn9W3CAi8SIyQ0TqXeqE/v7wv4+FF29+Ar6dDudL\nAXDmwhnumHwHry98vVD/FaN9yTZtC5u2Rf5I23chTYMGDViwYEGOymbk5+eXaVmOgiDAnScXkS7A\nIWNMvIhEk/WVwUogwhhzVkQ6AT8BV2V1vtjYWCIjIwEICwujXbsoypbtxJDXl0DLm6HkP1Adnp/7\nPHPnzeXpG5+mQ7sOgP2fJm3ZAD0uHMdpfKU+3jyOj4/3WvsXNumXs1i/fn2Oy+Ymz13i4uKcGwil\nfV/mllvHGETkv8D9QDJQHCgJTDXG9L7Ic3YC1xljjmdIz3bP53HjoN+gY5ju3aF6nDO9eZXm/Hj3\nj1QMrZjXt6KUykJhHGOYP38+vXr1Ys+ePXku6+fnx99//02NGjXyu5qZFJgxBmPM88aYCGNMDSAG\nmJsxKIhIeLqfm2IFq+PkQmwsTP26HIGTZ8GKB53pf+37i6afN2X1P6vz9D6UUgXLW2+9RY8ePVzS\nHnvsMR5//HHA2jKzXr16lCpVilq1ajF69Ohsz1W9enXmzp0LwPnz54mNjaVs2bI0aNCA5cuX57hO\nCQkJ9O7dmwoVKlC9enVee+01Z9727duJjo4mLCyMChUqcM899zjznnjiCcLDwyldujQNGzZ02XrU\nXdzalZQdEXkIMMaY0UB3ERkAXADOAXdfzjlvvx1+mxFEt9s+5cyR+tDxCfBLZW/CXm4aexNf3fEV\nd159Zz6+C++Ji4vTlTQdtC1svtYW8nL+dqOY4Tm/MomJieGVV17hzJkzlChRgtTUVKZMmcLPP/8M\nQHh4ODNnziQyMpKFCxdyyy230LRp00tunfnSSy+xc+dOdu7cyenTp7nllltyXKdBgwZx6tQpdu3a\nxZEjR+jQoQOVKlWib9++DB06lI4dOxIXF0dSUhIrVqwAYPbs2SxatIi///6bkiVLsmXLFsLCwnL8\nmpfLYxPcjDHz025JNcZ85ggKGGM+NsY0MMY0Msa0MMYsvdzXaNsW5s0Vyv09GL75Fc6XBuDshbPc\n9d1djFgwotBd9iqlMouIiKBx48b8+OOPAMyZM4cSJUrQpEkTADp16uTsf2/ZsiUdOnRg4cKFlzzv\nlClTePHFFyldujSVK1dm8ODBOapPamoqkydP5o033iAkJIRq1arx5JNP8tVXXwEQGBjI7t272b9/\nP0FBQc7lvwMDAzl16hQbN27EGEOdOnUIDw+/2EvliwI18zknmjSBhQuh8vkOMOYvOFbLmTd03lDu\nm3of5y6c82IN886X/ir0Nm0Lm7aFq3vuuYeJEycCMHHiRO69915n3q+//soNN9xAuXLlKFOmDL/+\n+mu223mmd+DAAZetQdNv13kxR48eJTk5mYiICJfn7t+/H7C6vlJTU2natCnXXHMNY8eOBaBNmzYM\nGjSIgQMHEh4ezsMPP5yjrUTzLG0bO19/WFXNuV27jKld2xiKHzP0bmt4CeejyegmZn/C/lydTymV\nWW7/X3rSkSNHTEhIiNm3b58JCwszmzdvNsYYk5iYaEJCQszUqVNNSkqKMcaY22+/3QwdOtQYY0xc\nXJypWrWq8zyRkZFmzpw5xhhjqlevbmbNmuXMGz16tEvZjETEbN++3aSkpJjg4GCzadMmZ95nn31m\n2rRpk+k5ixYtMsWKFTPbt2/P9H6io6PNsGHDsnyt7H4XjvRcfd8WuiuGNNWqwaJFEFWnLHz9Gywf\n4MxbfmA5TT9vysoDK71Yw8tX2G8VzA1tC5u2havy5cvTunVr+vbtS40aNahTpw4ASUlJJCUlUb58\nefz8/Pj111+ZPXt2js7Zs2dPXn/9dU6cOMG+ffv46KOPcvQ8Pz8/evbsyQsvvMDp06fZvXs3I0eO\npFevXgB8//33zquHsLAw/Pz88PPzY8WKFSxbtozk5GSKFy9OsWLF8PNz/9d2oQ0MABUqQFwctLwx\nEGb8D2Z8DKnWnqn7T+2n5diWTNkwxbuVVEq5zb333sucOXO47777nGmhoaGMGjWKHj16ULZsWSZN\nmsRtt92W7TnSz0UYPnw4ERERVK9enVtuuYXevbO98z7Tc0eNGkVISAg1atSgVatW3H///fTt2xeA\n5cuX06xZM0qVKsXtt9/OqFGjiIyMJCEhgf79+1O2bFmqV69O+fLlefrppy+3OXKsQO3HcLl1PXcO\nevaE6dOBGn9Ajx5Q/IQz/6XWLzGs9TCvTEZRqiArjPMYCqr8nMdQJAIDwIUL0LcvfPMNUG4r3NMV\nym915ves35Oxt40lJDAkH2qrVNGggcF3FJgJbr4kMBAmTIBHHwWOXWXdsbT9Zmf+dxu+o9XYVuxP\n2O+9SuaQ9iXbtC1s2hYqvxSZwADg5wcffADDhwPny1hzHZYOcuav/GclTT5vwvL9OZ/NqJRShU2R\n6UrKaNQoeOwxx8H1n0LnQeCXAkCxgGKMvW0sMQ1i8u31lCqMtCvJd+gYQz756itr3CElBag+F7m7\nO6bYv878F1u+yMttXsZPitSFlVI5poHBd+gYQz7p1Qt+/BGCg4GdbTGjl+F3vK4zf8TCEfSc0pMz\nSWe8V8ksaF+yTdvCpm2h8kuRDgwAXbvCrFlQsiRwvBapo5fgt6OjM/+HTT/QcmxL9p7c671KKuWj\nqlWrhojowwceOV2eIyeKdFdSeqtWwS23wJEjgF8yfrc8RWrTD5z54SXC+TnmZ5pVaea2OiilVH7T\nMYY82rIF2reHvY6LA7nuc6TrI6SSDECwfzCfd/2cXg17ubUeSimVX3SMIY/q1IE//4S6jmEGs7I/\nqWP/IIRyACSmJNL7p970/rE3CYkJXqun9iXbtC1s2hY2bYu80cCQQdWqsGABXHedI2F3a85+sIxy\nqfWcZb5a+xVRn0bx554/vVNJpZRyI+1KykZCAtx2m7UIHwBBp6j92KNsKzHeWcZP/Hih5QsMbTWU\nQP9Aj9VNKaVySscY8tn583D33TBtmp3WJPY7ttV5iBOJ9iJ8zSo34+s7v6ZW2VpZnEUppbxHxxjy\nWbFi8MMP0KePnbZ8XE/KTVlLkyuinWlL9y8l6tMoxq4e65HJPtp/atO2sGlb2LQt8kYDwyUEBMCX\nX8KQIXba9lVV2fB/f3BP+TcJ9LO6kM5cOEO/af3oMaUHx84e81JtlVIq77QrKRcmTICHH7b2d0jT\n87FVxNe8l63HtzjTKpWsxITbJ9CuRjsv1FIppWw6xuAB69bBXXfBtm12WpMWZ6kz+Em+3vypS9kn\nb3iS19q+RnBAsIdrqZRSFh1j8IBrroHly+GOO+y05YtD+G3QJ7xy9TSuCLnCmf7ukndpNqYZG49s\nzNc6aP+pTdvCpm1h07bIG48EBhHxE5FVIjItm/xRIrJNROJFJMoTdcqL0qWtQel33gF/awtpjh6F\n4TFd6Xt+LR1r3uIsu+bQGq4bfR0fLftIV6FUShUIHulKEpEngOuAUsaYbhnyOgGDjDFdRKQZ8IEx\npnkW5/CJrqSMFiywbmk9eNBO69TZ0Pqpjxn+51MkpiQ60zvX7syX3b4kPDTcCzVVShVFPtmVJCJV\ngM7AmGyK3AZMADDGLAVKi0iB+eZs1QpWr7b+TfPrTOHTfoOYcNNKrg2/1pk+c9tMrvnkGqZvne6F\nmiqlVM54oitpJPA0kN2f+5WB9Gta73ekFRgVK8KcOfDMM3barl3Qq2N9HpRlPNHcvtf1yNkjdJ3Y\nlUdmPMLZC2cv6/W0/9SmbWHTtrBpW+RNgDtPLiJdgEPGmHgRiQZydTmTUWxsLJGRkQCEhYURFRVF\ndHQ0YH8QvHW8aFEcnTpB8+bRxMZCQkIcSUkw6OFo+vR5l/9GV+LdZa9zLNya4/DJlE+YPns6Pz/7\nM42ubOT1+hfU4zS+Uh9vHsfHx/tUfbx5HB8f71P18eRxXFwc48aNA3B+X+aWW8cYROS/wP1AMlAc\nKAlMNcb0TlfmU2CeMWay43gz0NoYcyjDuXxyjCEr27ZB9+6wdq2ddu21MObbY7yx4UGmbprqTA/0\nC2RE2xE81eIp3UJUKZXvfHoeg4i0Bp7MYvC5MzDQMfjcHHi/IA0+Z+fsWRgwwJoUl6ZUKRg3zvBv\n5FgG/zqYMxfsLUPbRLZh/O3jqVq6qhdqq5QqrHxy8DkrIvKQiDwIYIyZCewUkb+Bz4BHvFGn/BYS\nAuPGwWefQVCQlZaQAHfeKWyZ1I/lD6ymaeWmzvLzds3j2k+vZcqGKZc8d8ZulKJM28KmbWHTtsgb\njwUGY8z8tKsFY8xnxpjR6fIGGWNqGWMaGmNWeapO7iYCDz4IixdD+q6+t96CAT1r832nRbzY8kVn\nF9KJ8yfo+X1PYn+K5VTiKe9UWilV5OmSGB5y/Dj06gUzZ9ppFSvCd9+BVFtErx97sevELmdejTI1\n+PqOr7mh6g2er6xSqtAoMF1JRVHZsvDLLzBihHUlAdakuDZtYOmUm1j9YDz3X3u/s/yOf3fQcmxL\nXop7ieTUZC/VWilVFGlg8CA/P3jhBZg1C8qXt9JSUuCpp+A/95fmo7Zf8e2d31I6uLSVZ1J4ef7L\ntBrbih3/7nCeR/tPbdoWNm0Lm7ZF3mhg8IL27WHVKmie7t6rqVOhSRNowD2seXgNrarZU6mX7FtC\nw08bMj5+vK63pJRyOx1j8KKkJOtq4cMP7bTixa07me69L4W3/nyLYXHDXLqSetTrwWe3fkaZ4mW8\nUGOlVEHj0/MY8qowBoY0kybBAw/AGXtaAw8/DO+/D+uOreC+qfex9dhWZ16VUlX4pMsn3HrVrV6o\nrVKqINHB5wIqJgaWLYO6de20Tz+Fm26C8knXs+rBVTzY+EFn3r41++g6sSt3TL6D3Sd2e6HGvkP7\nkm3aFjZti7zRwOAj6tWzgsPdd9tpK1bAddfBwrkl+KzrZ/x090+UDynvzP9p80/U+1893lz0Jkkp\nSV6otVKqMNKuJB9jDHz0EQwZAsmOoQURGDoUhg2DfxOP8uwfz/LF6i9cnnd1+av5X5f/ER0Z7flK\nK6V8lo4xFCKLF0PPnrB/v53WoQN88411q+vivYsZMGMAaw+tdXne/dfez9vt36ZiaEUP11gp5Yt0\njKEQadHCuqW1XTs7bfZsaNwY/ve/OFpUbcHKB1cysuNIQoNCnWW+Xvs1dT+qy8fLPiYlNcULNfcs\n7Uu2aVvYtC3yRgODD6tQwZoM98ILdtrevTB4MLzxBpAawOPNH2fzwM3cXd8enDiZeJJBvw6i6Zim\nLNu/zPMVV0oVaNqVVEBMn26ttXTihJ3WpAl8+SU0aGAd/779dwbOHMi249ucZQThoese4r/t/qtz\nH5QqgnSMoZDbudO6a2n5cjstKMgalH7mGQgMhMTkRN5e/DavLXyN88nnneWuCLmCt9u/Te+GvRHJ\n00Z6SqkCRMcYCrnq1a1B6f7945x7PCQlwYsvWstrrF0LwQHBvNjqRTY8soHOtTs7n3vk7BFif46l\n9bjWrD+83kvvIP9pX7JN28KmbZE3GhgKmIAAuPdeWL0amtr7/LBqFVx/PbzyCly4YC3bPf2e6Uzt\nOZWqpexd4RbuWUjUp1E8PftpTied9sI7UEr5Ou1KKsCSk2HkSGuOQ2KinR4VBWPHWv8CnE46zavz\nX+W9v95zWXepSqkqfHDLB9xR9w7tXlKqkNIxhiJq82bo1w+WLLHTAgLg+eetO5rSup02HN7AIzMf\nYcHuBS7P71SrEx92+pCaZWt6sNZKKU/QMYYiImP/ad26sHAhvPsuFCtmpSUnW91K119vdTMB1K9Q\nn7g+cUy4fQIVSlRwPv/Xv3+l/v/q88r8V1wGrAsC7Uu2aVvYtC3yRgNDIeHvby2jsWaNtfhemnXr\nrLGIF1+0uptEhF4Ne7F54GYeuf4RBOsPicSURIbHDeeaT65h9vbZXnoXSilfoF1JhVBqqrXe0rPP\nwrlzdnr9+tbYQ5MmdtqKAysYMGMAKw6scDlHj3o9GNlxJJVLVfZQrZVS7qBjDMrF33/Df/4DC9IN\nKfj5wdNPw0sv2d1OKakpjF45mufmPMfJxJPOsqFBobwc/TKPNn2UQP9Az1ZeKZUvdIyhiMhp/2mt\nWjBvnrVDXIkSVlpqKrz5JjRqBH/9ZaX5+/kzoMkAtgzaQu+GvZ3PP510midnP8l1o6/jzz1/5vO7\nyB/al2zTtrBpW+SNWwODiASLyFIRWS0i60RkeBZlWovICRFZ5Xi86M46FTV+fjBokDX5rU0bO33z\nZrjxRuvqIa27KTw0nPG3jyeuTxz1rqjnLLvu8DpuGnsT/X7ux5EzRzz8DpRSnub2riQRCTHGnBUR\nf+BPYLAxZlm6/NbAk8aYbpc4j3Yl5VFqKowebQWD0+nmtl11lTX20KKFnZaUksT7f73Py/Nf5uyF\ns870MsXK8MbNb/BA4wfwE73gVMrX+WRXkjEm7VslGAgAsvp219lVHuDnZ+0lvX493Hyznb51q3Un\n05AhcNbx2wryD+KZG59h08BN3FH3DmfZf8//y0PTH6LFFy1YeWClh9+BUsoT3B4YRMRPRFYDB4Hf\njTHLsyh2g4jEi8gMEamXRb5KJ6/9p9WqWXs7jB4NJUtaacZYs6gbNrTmRKSJKB3B1LunMv2e6VQP\nq+5MX7p/Kdd/fj09p/Rk89HNeapPXmhfsk3bwqZtkTcB7n4BY0wq0EhESgE/iUg9Y8zGdEVWAhGO\n7qZOwE/AVVmdKzY2lsjISADCwsKIiooiOjoasD8Iepyz4/nz46hdG9avj+bBB2HWLCv/77+jad0a\n7rgjjgcegE6drPIlDpTgk/qf8Kf/n7z555sk/W3tMT2FKfyw6Qfa+7UnNiqWmFtjPPp+0ni7PX3h\nOD4+3qfq483j+Ph4n6qPJ4/j4uIYN24cgPP7Mrc8eruqiAwFzhhj3rtImZ3AdcaY4xnSdYzBTYyx\nxhieeAISEuz0GjWs/R5at3Ytv/XYVp6b8xxTN011SQ/0C+Sh6x7ihVYv6NaiSvkInxtjEJHyIlLa\n8XNxoD2wOUOZ8HQ/N8UKVi5BQbmXiLXW0oYN0NleqZsdOyA62rqryWWwutxV/NDzB5Y9sIwONTs4\n0y+kXuCj5R9R44MaPPfHcxw/p79GpQoid48xXAnME5F4YCkwyxgzU0QeEpEHHWW6i8h6xzjE+8Dd\n2Z1MWTJ2o+SXKlWsneLGjYOwMDv944/hmmtg7lzX8k0qN2HW/bOI6xPHjVVvdKafSz7HG3++QfUP\nqjNiwQi3Lu/trrYoiLQtbNoWeePWwGCMWWeMaWyMiTLGXGuMec2R/pkxZrTj54+NMQ2MMY2MMS2M\nMUvdWSd1cSLQp4919XDrrXb6rl3Qrp11V1P67iaA1pGtWdh3ITPunUFUxShnekJiAkPnDaXGBzV4\n/6/3C9wCfUoVVbokhsqWMfDNNzB4MPz7r50eEQFjxkD79pmfk2pS+X7j9wydN5Stx7a65FUpVYXh\nrYfTp2EfXWJDKQ/RtZKUWxw8CAMGwE8/uabHxsJ//wtXXpn5OcmpyXy15itemv8Se07uccmrXbY2\nr7R5hZ71e+okOaXczOcGn5V7eLr/tGJFmDoVJk6EcuXs9HHjoHZtKzikX8UVIMAvgL6N+rJ10FZG\n3TLKZf+Hbce3cc8P99Dos0b8suUX8hLwtS/Zpm1h07bIGw0MKkdEICbGGnu46y47/cwZa5e4unVh\n0iSr+ym94IBgHm32KDsG7+C/bf9LWDF7VHvtobV0m9SNFl+2YN7OeR56J0qpS8lRV5KIPAaMBU4B\nY4BGwLPGGI/t6KJdSb5l9mxrCY0NG1zTW7SwZlA3bZr1806cP8E7i9/h/b/e58yFMy55N9e4mdfa\nvkbTytk8WSmVa24bYxCRNcaYhiLSEXgIGAp8ZYxpfHlVzT0NDL4nOdkahB46FI4edc277z54/XWo\nWjXr5x46fYjXF73OJys+ISklySXvtjq38WqbV7km/Bo31VyposOdYwxpJ+2MFRA2oAvfeY2v9J8G\nBFi3r27bBk89BYHpbjT65huoUweGD7e6mzIKDw3n/VveZ9uj2/hPo//gL/7OvJ+3/EzDTxty/9T7\n+fv43xetg6+0hS/QtrBpW+RNTgPDShGZjRUYZolISSDVfdVSBUlYGLz9NmzcCHfYC7Fy7hy88oq1\nrPeECday3xlFlI5gTLcxbBy4kZgGMc50g+Gbdd9w9cdX8/D0h9mfsN8D70QpBTnvSvIDooAdxpgT\nIlIWqGKMWevuCqarg3YlFRBxcdb4w+rVrunXX2+NP9x0U/bPjT8Yz9B5Q5m+dbpLerB/MAObDOTZ\nm57lihJX5H+llSqk3DnGcCMQb4w5IyL3A42BD4wxuy+vqrmngaFgSUmxrhKef96aB5Fejx7W9qLV\nq2f9XIBIUzIAAAAgAElEQVTFexfz/Jznmb97vkt6aFAoQ5oPYcgNQyhdrLQbaq5U4eLOMYZPgLMi\n0hB4EtgOTMhl/VQ+KQj9p/7+0LevtQnQCy9AsWJ23pQpcPXV8NxzmZfXSNOiagvm9ZnH7Ptn06RS\nE2f66aTTvLLgFWqMqsHbf77Nb3/85uZ3UnAUhM+Fp2hb5E1OA0Oy48/124CPjDEfAyXdVy1VWJQs\nCSNGWHtMx9hDCCQmwhtvWBPkPv/cusLISERoX7M9Sx9Yyo93/0j9K+o7846fO84zfzzDvT/cy5uL\n3uTfc/9mPoFS6rLktCtpPvAb0A9oCRwG1hhjPHY/oXYlFQ5Lllj7PizNsFTitdda4w9t22b/3JTU\nFCaun8jwuOHs+HeHS16JwBI80PgBHm/+OJFhkflfcaUKKHeOMVQE7gWWG2MWikgEEG2M8Vh3kgaG\nwiM11Vpe49lnYd8+17xu3eCdd6wriewkpSTx5eoveXXBqxw4dcAlz0/86F6vO0/e8KROlFMKN44x\nGGMOAt8ApUXkVuC8J4OCclXQ+0/9/KwJcFu2WLezhoTYedOmQf361l1N/2bTOxTkH8TD1z/MjsE7\n+L/K/+fSxZRqUvluw3c0G9OMlmNb8vPmn0k1RePO6oL+uchP2hZ5k6PAICI9gWVAD6AnsFREuruz\nYqrwCwmxZk1v3WrtAZHmwgWrW6l2bfjoI+s4K8EBwdxS6xbWDVjHb/f9xs01bnbJX7RnEbdPvp26\nH9Xl0xWfcvbCWTe+G6UKjxwviQG0N8YcdhxfAfxhjGno5vqlr4N2JRVyK1ZY4w+LFrmmX301vPsu\ndOp06XOsObiGd5e8y8T1E0lOTXbJK1e8HAObDGRg04Euq70qVZi5c4xhXfqBZseENx18VvnOGPjh\nB3jmGdi50zWvY0crQNSvn/Vz09uXsI8Pl37IZys/42TiSZe8YP9gejfszZAbhlC3fN18rL1Svsed\n8xh+E5FZIhIrIrHADGBmbiuo8kdh7j8Vge7dreU13nzTut01zaxZ0LAhDBxoL9qXXVtUKVWFN9u/\nyd4n9jKy40iqla7mzEtMSeTzVZ9z9cdXc+u3txK3Ky5Pe0L4isL8ucgtbYu8yeng89PAaOBax2O0\nMeb/3FkxVbQVK2ZdNWzbBg8+aA1YgzXf4X//g1q14L33sh9/SFMyuCSPN3+cvwf/zeTuk10mywHM\n2DaDNuPbcP3n1/Ptum+5kHKJEypVBOjWnqpAWLvWulNpzhzX9Fq1rAHse++1Vnu9FGMMi/Ys4t0l\n7zJtyzQMrp+pqqWq8njzx3mg8QOUCi6Vj+9AKe/I9zEGETkFZFVAAGOM8dj/HA0MyhiYPt1a4nvr\nVte8GjWsdZl69YKgoJydb+uxrYxcMpJxa8ZxPvm8S16p4FL0b9yfx5o9RtXS2WwqoVQB4LbBZ1+g\ngcEWFxdHdHS0t6vhNUlJVnfSyy/DiRNxQLQzLyLCmjjXrx8EB+fsfEfOHOGTFZ/w0bKPOHL2iEte\ngF8APev35MkbnqTxlR7bl+qyFPXPRXraFjZ3Dj5fFhEJFpGlIrJaRNaJyPBsyo0SkW0iEi8iUe6s\nkyr4goLg8cetu5b69oWyZe28PXvgkUesK4hRo6w9IS7lihJXMKz1MHY/vpvRt46mTrk6zrzk1GS+\nXfct142+jrbj2zJj64wiM2FOFV1uv2IQkRBjzFkR8Qf+BAYbY5aly+8EDDLGdBGRZljLeTfP4jx6\nxaCydOqUdQXx7rtwxPUPfsLDra6nhx+G0NCcnS/VpDJz20zeWfxOpmW/Aa4ufzVDbhjC/dfeT7GA\nYlmcQSnf4dNdSSISAiwABhhjlqdL/xSYZ4yZ7DjehLUO06EMz9fAoC7qzBkYPRreeivzHhDlylmD\n14MGQalcjIytOLCCd5e8y5QNU0gxrkvAVihRgUFNBjGgyQDKh5TPh3egVP7zua4ksCbDichq4CDw\ne/qg4FAZ2JvueL8jTWVD79G2pW+LEiWsmdM7d1pLaVSpYpc7dszaF6JaNWtsIrt1mDK6vtL1TLxr\nItsHb2dI8yGUDLInVhw+c5hhccOIGBlB/2n9WbZ/mVfnQ+jnwqZtkTc5uMEvb4wxqUAjESkF/CQi\n9YwxGy/nXLGxsURGRgIQFhZGVFSUc4Ap7YOgx0XrOE36/GLFoH79OL74Anbtiub112HXLiv/xIlo\nXnoJ3nwzjjvvhPffj6Z8+Uu/3s74nXQN7sqwJ4bx+arPefObNzl65ihUh3PJ5xgzdQxjpo7hmmbX\n0L9xf6qdqEap4FIebY/4+Hiv/z585Tg+Pt6n6uPJ47i4OMaNGwfg/L7MLY/elSQiQ4Ezxpj30qVl\n7EraDLTWriSVXy5cgG++gf/+15owl16JEjBggDUOER6ei3OmXOC7Dd/xzpJ3iD8Ynyk/2D+Y7vW6\n80DjB2hdrTUiubqSVyrf+NwYg4iUBy4YY06KSHFgFvCGMWZmujKdgYGOwefmwPs6+KzcITkZvvvO\n2lFu0ybXvGLFrBnWzzwDlXPRkWmMYfHexYxZPYbJ6ydzLjnzbVC1y9bmP43+Q2xULOGhuYg+SuUD\nXwwM1wDjscYy/IDJxpjXROQhrAlyox3lPgJuAc4AfY0xq7I4lwYGhzi9R9vpctoiNdVaqG/ECGtG\ndXpBQfCf/8D//Z81HpEbJ8+fZOL6iXy+6nNW/ZPpI0yAXwBdr+pK/8b96VCzA/5+/rl7gUvQz4VN\n28Lmc4EhP2lgsOmH3paXtkhNhV9+gVdfhZUrXfMCAqw9Ip57DmrWzP25V/2zijGrxvDNum9ISEzI\nlF+1VFX6NepHv0b9iCgdcVn1z0g/FzZtC5sGBqUugzHw22/WbnJ//eWa5+9v7Tb3/PNQp07Wz7+Y\nsxfOMmXDFMasHsOiPYsy5QtCx1od6d+4P12v6kqgf+BlvgulsqaBQak8MMZapO/VV2HBAtc8Ebj7\nbuuW1wYNLu/8m45s4ovVXzB+zXiOnj2aKb9CiQrENozlP43/w1Xlrrq8F1EqA5+cx6DyX8ZbNYuy\n/GwLEbj5Zpg/33rcnG6nUGNg0iS45hq46y5YvTr357/6iqt5p8M77HtiH991/472Ndq75B8+c5i3\nFr9FnY/qED0umq/Xfs25CzlY08NBPxc2bYu80cCgVBZatYLff4fFi6FzZ9e8qVOhcWPo2hWWLcv6\n+RcTHBBMj/o9mN1rNjsG7+DFli9SqWQllzLzd8+n14+9qPReJR6d+ShrD63N5mxK5T/tSlIqB1as\nsO5i+vnnzHkdO1orurZubV11XI7k1GR++/s3xqwaw/St0zMtvwHQpFIT+jfuT0yDGEoGl8ziLEpl\npmMMSrnZ2rVWgPj+e6t7Kb1rrrHWYrrvPmvi3OU6cOoA4+PHM2b1GHb8uyNTfonAEsQ0iKF/4/40\nrdxUJ8+pi9LAUETorXg2b7XFxo3WTOqJE63bXtMLC7P2gxg40Fr++3KlmlTm7ZzHmNVjmLppKkkp\nSZnKNKjQgP6N+3P/tfezdula/Vw46P8Rmw4+K+Uh9erB11/D5s3WkhrprxBOnLD2o65VyxqHmD07\nc/DICT/xo12Ndky8ayL7h+xnZMeR1LuinkuZ9YfX89hvj1Hp3Uq8Ov9Vpm+dnmUAUSo39IpBqXxw\n4gSMH2+t6vr335nzr7rK6mbq0yd3y35nZIzhr31/8fmqz5m8YTJnL5zNVKZMsTLcdfVdxDSIIToy\nOt9nWKuCRbuSlPKy1FSYNQs+/BB+/TVzfmioFRwGDYK6dfP2WgmJCUxcN5Exq8ew4sCKLMuElwin\nR70e3HPNPTSv0hw/0U6CokYDQxGh/ac2X26LbdusneW+/BISMq+KQfv28Oij1u2w/nn8o37NwTW8\n+c2bLPZfzO6Tu7MsE1E6grvr301MgxgaVWxUqAetfflz4Wk6xqCUD6ldG0aOhP37rQBRz3V4gN9/\nh27drHLvvAPHj1/+azWs2JAHr3uQnY/tZHG/xQxuOpiKoRVdyuw5uYe3F7/NdaOvo+7HdRk+bzib\njmzK5oyqKNMrBqU8xBiYN8/qZpo2LfOAdPHi1q2ujz4K116b99dLSU1hwe4FTFw/kR82/cDxc1lH\nnmvDryWmfgwxDWKoXqZ63l9Y+RTtSlKqgNi1Cz75BMaMyfpKoVUrK0Dcfru10mteJaUk8ceOP5i0\nfhI/bv6R00mnsyzXrHIzYhrE0LN+z0yzsVXBpIGhiND+U1tBb4tz56y5EB9+CPGZN4KjShV4+GHo\n3x8qVLj4uXLaFucunGPmtplM2jCJ6Vuncz75fKYygtA6sjUx9WO4q95dlA8pn8N35BsK+uciP+kY\ng1IFTPHi1mS4Vatg4ULo2dN1IHrfPnjxRaha1bqbafnyfHjNwOLcVe8upvSYwuGnDvP1HV9z61W3\nEuBnX5oYDHG74nh4xsNUfKcinb7pxPj48Zw8fzLvFVA+T68YlPIx+/fDZ59Zj8OHM+c3a2Z1M3Xv\nDsHB+fe6x88dZ+qmqUxaP4l5u+aRajLPygv2D6Zz7c7ENIjh1qtuJSQwJP8qoNxCu5KUKkQSE2HK\nFKubKatVXMPDrX2qH34YKuXzcMDB0wf5fuP3TFw/kcV7F2dZpkRgCbrV6cY9De6hQ80OBAfkY5RS\n+UYDQxGh/ae2otIWy5ZZs6onT4akDCteBARYe0S0aBHHo49GX/YKr9nZfWI33234jkkbJmW5lzVA\nWLEw7qx7JzENYmhTvY1Lt5Q3FJXPRU5oYCgi9ENvK2ptcfgwjB5t3dF04EDG3Dhq146mTx/o1Qsi\n8mcraRdbjm5h8obJTFw/kc1HN2dZpkyxMnS5qgvdrupGx1odKRWchzVALlNR+1xcjAYGpYqICxfg\np5+sbqaFCzPni0CbNhAbC3fembdlwLNijGHd4XVMWj+JSesnsfPEzizLBfoF0qZ6G7pd1Y2udboS\nUdoN0UpdlAYGpYqgNWvg44+trUdPncqcHxpqDVTHxkLLluCXz/ciGmNYtn8Zk9ZP4vtN37MvYV+2\nZRuGN6RbnW50q9ONxlc21rWbPEADQxGhl8k2bQvbb7/Fcfx4NOPGwR9/ZN5ICCAyEnr3th41a+Z/\nHYwxxB+MZ9qWaUzbOi3bMQmASiUr0fWqrnSr04221dtSLKBYvtVDPxc2nwsMIlIFmACEA6nA58aY\nURnKtAZ+BtK2qppqjBmRxbk0MDjoh96mbWFL3xb79ln7RYwfb+0ZkZWWLa25ET165G0p8IvZe3Iv\n07dOZ9rWaczdOTfbvSJKBJagQ80OdKvTjS61u3BFiSvy9Lr6ubD5YmCoCFQ0xsSLSCiwErjNGLM5\nXZnWwJPGmG6XOJcGBqVyyRjrjqbx462upn//zVymeHG44w6rq6lt27yv9JqdU4mnmL19NtO2TmPG\n1hkcO3csy3KC0KJqC2eXU51ydQr1SrDu5nOBIdOLifwEfGiMmZMurTXwlDGm6yWeq4FBqTxITIRf\nfoFx4+C33yAlJXOZypWtO5r69Mn7fhEXk5yazJK9S5xdTluPbc22bO2ytZ1BokXVFl6/Fbag8enA\nICKRQBzQwBhzOl16a+AHYB+wH3jaGLMxi+drYHDQy2SbtoUtN21x8CB8+60VJNaty7pMs2ZWgIiJ\ngTJl8q2aWdpydIszSCzeuzjLWdcAZYuXpUvtLnSr042ONTtSMrhkluX0c2Hz2cDg6EaKA141xvyc\nRV6qMeasiHQCPjDGXJXFOUyfPn2IjIwEICwsjKioKOcvPy4uDqBIHKf97Cv18eZxWpqv1Mebx/Hx\n8Tz++OO5en7r1tHEx8Nrr8Xx+++QkGDlW/9dAaIJCoIbboijY0d4+uloAgLc+36OnDnCuxPfZcne\nJawMXsmZC2cg7W7YtFXBd0KAfwDt2rSjW51ulD9UngqhFZzne//994v098O4ceMAiIyM5OWXX/a9\nwCAiAcB04FdjzAc5KL8TuM4YczxDul4xKOVGFy5Y25GOGwfTp1vHGYWHw/33W1cS11zj/jqdTz5P\n3K4462piyzT2n9qfbdmoilF0u8q+FVbHJSw+ecUgIhOAo8aYIdnkhxtjDjl+bgp8Z4yJzKKcBgal\nPOToUWuwetw4WLky6zKNGlkB4t574Yq83USUI8YYVh9c7QwSqw+uzrbslaFX0qFmB9rXaM/NNW4m\nPDTc/RX0UT4XGETkRmABsA4wjsfzQDXAGGNGi8hAYABwATgHPGGMWZrFuTQwOMRp/6mTtoXNXW2x\nYYN1V9NXX1ljExkFBECXLlaQ6NIFgoLyvQpZ2nNyj3Ur7BbrVtgLqekucXZidzth7VLXoUYH2tds\nT8uIlhQPLO6ZSvoAnwsM+UkDg02/DG3aFjZ3t0VysrVP9fjx1nIciYmZy5QrZ82L6N4dWrfOn93n\nciIhMcG6FXbLNGZsm8HxjcddAkN6wf7B3BRxk/OKomHFhoV6BrYGBqWUR5w4Ya30On48LFmSdZly\n5az5Ed27W/MjAgM9U7eU1BRWHFjB7zt+5/cdv7N472KSU5OzLX9FyBW0q9HOeUVRpVQVz1TUQzQw\nKKU8butWmDDBeuzdm3WZMmXgttusIHHzzfm7wdClnEo8xfzd8/l9uxUoNh3ddNHydcvXpX2N9rSv\n0Z7oyOhsb4ktKDQwFBHafWLTtrB5uy1SU2HRIvjhB/j++6yWBbeUKgXdullBokMHa+Z1frtYW+xL\n2McfO/5g9vbZ/LHjD46cPZLteQL8Arihyg1WoKjZnusrXV/gJthpYCgivP0F4Eu0LWy+1BapqfDX\nX3aQ2LMn63KhoXDrrVaQ6NQJQvJpp9CctkWqSWXtobX8vv13Zu+YzcLdC0lMyWLwxCGsWBhtq7d1\nXlHULOuGlQjzmQYGpZTPMQZWrLACxJQpsDPrrRsICYHOna0g0aWLFTQ87dyFcyzas4jfd/zO7O2z\nWXNozUXL1yhTwxkk2lZvS5nibp4ifhk0MCilfJoxEB9vB4lt27IuV6wY3HKLFSRuvRVKl/ZsPdMc\nOn2IOTvnOAPFgVPZ9I8BfuJHk0pNnN1Ozas0J8jfQ/fuXoQGhiLCl7oMvE3bwlbQ2sIYWL/eDhKb\nshkTDgqyxiK6d7fGJnKybpM72sIYw6ajm5yD2HG74qzlOrIRGhTKjVVvpFW1VrSq1oomlZoQHODB\nUXcHDQxFREH7AnAnbQtbQW+LjRutIPH999kv7BcQYN3V1L27dZdT+fJZl/NEWySlJLFk7xLnbbEr\nDqzIdvE/sOZPNKvSjFYRVqC4oeoNhAa5v79MA4NSqlDYutUeuF6VzSZw/v7Wvtbdu1vzJSpU8Gwd\nMzp+7jhzd851XlFktw92Gn/xp/GVjZ1XFDdF3ETZ4mXzvV4aGJRShc6OHXaQWLYs6zJ+ftCqlR0k\nKlXybB2zsvPfnSzcs5AFuxewYPcCth3PZkAlnQYVGjivKFpWa0mlknl/IxoYioiC3mWQn7QtbEWh\nLXbvhqlTrSCxeHHWZUSgXr047rsvmi5drFVgfWGh1YOnD7JwtyNQ7FnAukPrMFz8O61mmZrOK4pW\n1VpRPax6rleN1cBQRBSFL4Cc0rawFbW22LcPfvzRChILF1qD2bY4IBqAKlWs22A7d4Z27bxzG2xW\n/j33L3/u/dN5RbHyn5UXXboDoFLJSlaQcFxVXH3F1Zdc50kDg1KqSPrnHztIzJ9vTbDLSlAQREdb\nQaJLF6hVy6PVvKgzSWf4a99fziuKv/b9xfnk8xd9TtniZWkZ0dJ5RRFVMSrTzGwNDEqpIu/IEWtP\n6xkzYNYsa8G/7NSubQWIzp2tMQpPruF0KYnJiaz8Z6XzimLRnkWcSjp10eeEBoXSomoL5xVFk8pN\nKB5YXANDUVDUugwuRtvCpm1hS2uL5GRr9deZM61Akd1tsAAlSli3wqYFisqVPVffnEhJTWHNoTXW\nOMUeK1gcPXv0os8J9g8mcWhirgNDwVoNSimlciEgAFq2tB6vv26t/jpzpvX44w84e9Yue+YM/Pyz\n9QBo2NAOEs2bW7fHepO/n3V7a+MrG/NY88cwxrD56GbnnU/zd89nX8I+l+dcbN2ni9ErBqVUkXT+\nvDUekXY1sX179mXLloWOHa1A0bFj9hPrvMkYw+6Tu13ufNp6bCu8hHYlKaVUbhljrds0Y4YVKObP\nhwsXsi7r5wfNmtlXE1FRvnE7bFYOnj7IlSWv1MBQFGhfsk3bwqZtYctrW5w6BXPm2IEiu70lAK68\n0r7L6eaboaSP7etzOXclFd6NTpVS6jKVLAm33w6ff27Nl1i9GkaMgBYtrCuG9P75B774Au6809rO\n9OabYeRI2LIl49yKgkOvGJRSKheOHbNug50xw7ot9vjx7MtGRFiT6tq2tR7eWKpD5zEopZQHpaTA\n0qX2AHZ8/MXL161rB4roaGtQ2900MBQR2pds07awaVvYvNUW+/fDr7/at8Oeush8NBFo1MgKEu3a\nwU03uWe5Dp8bYxCRKiIyV0Q2iMg6ERmcTblRIrJNROJFJMqddVJKKXepXBkeeMBa6O/YMWuhvxEj\nrOXBM86qNsZaUvydd6z9rsuUseZbDB8OCxZA4uVNQcgXbr1iEJGKQEVjTLyIhAIrgduMMZvTlekE\nDDLGdBGRZsAHxpjmWZxLrxiUUgXWuXPWLOw5c2DuXFi+3OqKyk7x4lagSLuiaNTo8ibZ+XxXkoj8\nBHxojJmTLu1TYJ4xZrLjeBMQbYw5lOG5GhiUUoXGyZPWqrBpgWLt2ouXDwuzxiXSBrLr1cvZ/Amf\n60pKT0QigShgaYasysDedMf7HWkqG3Fxcd6ugs/QtrBpW9gKQluULg233mrd2rpmDRw6BJMnw4MP\nZr3q64kT8NNPMHgwNGhgzZ+4917rVtmdF98sLtc8slaSoxvpe+AxY8zpyz1PbGwskZGRAISFhREV\nFeUcYEr7IOhx0TpO4yv18eZxfHy8T9XHm8fxjtuDfKU+OT3u2TOanj2t40OH4Ny5aObMgV9/jePY\nMUjbYwKs/IkTo5k40TquWBG6dInmyivj2Lp1HMWL4/y+zC23dyWJSAAwHfjVGPNBFvkZu5I2A621\nK0kppSzGWPtgp3U7zZt38fkTAPXrW11OH37og2MMIjIBOGqMGZJNfmdgoGPwuTnwvg4+K6VU9lJT\nre6ntECxYIG1OmzWfGyMQURuBO4D2orIahFZJSK3iMhDIvIggDFmJrBTRP4GPgMecWedCoOM3ShF\nmbaFTdvCVtjbws/PukvpqaesORPHj8OiRfDyy9C6tbVTXV64dYzBGPMncMkbrIwxg9xZD6WUKsyC\nguDGG63HsGHWPhN//mldUbz5Zu7PpzOflVKqEPPp21WVUkoVDBoYCqDC3n+aG9oWNm0Lm7ZF3mhg\nUEop5ULHGJRSqhDTMQallFJ5poGhANL+U5u2hU3bwqZtkTcaGJRSSrnQMQallCrEdIxBKaVUnmlg\nKIC0/9SmbWHTtrBpW+SNBgallFIudIxBKaUKMR1jUEoplWcaGAog7T+1aVvYtC1s2hZ5o4FBKaWU\nCx1jUEqpQkzHGJRSSuWZBoYCSPtPbdoWNm0Lm7ZF3mhgUEop5ULHGJRSqhDTMQallFJ55tbAICJf\niMghEVmbTX5rETkhIqscjxfdWZ/CQvtPbdoWNm0Lm7ZF3rj7imEs0PESZRYYYxo7HiPcXJ9CIT4+\n3ttV8BnaFjZtC5u2Rd64NTAYYxYB/16iWK76vhScOHHC21XwGdoWNm0Lm7ZF3vjCGMMNIhIvIjNE\npJ63K6OUUkVdgJdffyUQYYw5KyKdgJ+Aq7xcJ5+3a9cub1fBZ2hb2LQtbNoWeeP221VFpBrwizHm\n2hyU3QlcZ4w5nkWe3quqlFKXIbe3q3riikHIZhxBRMKNMYccPzfFClSZggLk/o0ppZS6PG4NDCLy\nLRANlBORPcBwIAgwxpjRQHcRGQBcAM4Bd7uzPkoppS6twMx8Vkop5Rm+cFfSJYnILSKyWUS2isj/\nebs+3iIiVURkrohsEJF1IjLY23XyJhHxc0yMnObtunibiJQWkSkissnx+Wjm7Tp5g4g8ISLrRWSt\niHwjIkHerpMnZTWpWETKiMhsEdkiIrNEpPSlzuPzgUFE/ICPsCbK1QfuEZG63q2V1yQDQ4wx9YEb\ngIFFuC0AHgM2ersSPuIDYKYx5mqgIbDJy/XxOBGpBDwKNHbc7BIAxHi3Vh6X1aTiZ4E/jDF1gLnA\nc5c6ic8HBqApsM0Ys9sYcwGYBNzm5Tp5hTHmoDEm3vHzaaz//JW9WyvvEJEqQGdgjLfr4m0iUgpo\naYwZC2CMSTbGJHi5Wt7iD5QQkQAgBDjg5fp4VDaTim8Dxjt+Hg/cfqnzFITAUBnYm+54H0X0yzA9\nEYkEooCl3q2J14wEngZ0kAyqA0dFZKyja220iBT3dqU8zRhzAHgX2APsB04YY/7wbq18QoW0uz+N\nMQeBCpd6QkEIDCoDEQkFvgcec1w5FCki0gU45Lh6yvZ26CIkAGgMfGyMaQycxeo+KFJEJAzrr+Nq\nQCUgVETu9W6tfNIl/5gqCIFhPxCR7riKI61Iclwifw98ZYz52dv18ZIbgW4isgOYCLQRkQlerpM3\n7QP2GmNWOI6/xwoURc3NwA5jzHFjTAowFWjh5Tr5gkMiEg4gIhWBw5d6QkEIDMuBWiJSzXGHQQxQ\nlO9C+RLYaIz5wNsV8RZjzPPGmAhjTA2sz8NcY0xvb9fLWxzdBHtFJG05mXYUzUH5PUBzESkmIoLV\nDkVuEJ7MV9HTgFjHz32AS/5B6e21ki7JGJMiIoOA2ViB7AtjTFH8ZSMiNwL3AetEZDXWJeHzxpjf\nvFsz5QMGA9+ISCCwA+jr5fp4nDFmmYh8D6zGmjS7Ghjt3Vp5VjaTit8ApohIP2A30POS59EJbkop\npdIrCF1JSimlPEgDg1JKKRcaGJRSSrnQwKCUUsqFBgallFIuNDAopZRyoYFBKQ8QkdYi8ou366FU\nTvXBgW4AAAGuSURBVGhgUMpzdNKQKhA0MCiVjojcJyJLHauUfuLYDOiUiLzn2ADmdxEp5ygbJSJL\nRCReRH5I2wBFRGo6ysWLyAoRqe44fcl0m+l85bU3qdQlaGBQysGx6dHdQAvHKqWpWEuQhADLjDEN\ngAVYywyAtbb908aYKGB9uvRvgA8d6S2AfxzpUVhLV9QDaoqILvCmfJLPr5WklAe1w1qVdLljEbZi\nwCGsAPGdo8zXwA+OzXFKOzZGAStIfOdYEr2yMWYagDEmCcA6HcuMMf84juOBSGCxB96XUrmigUEp\nmwDjjTEvuCSKDM1QzqQrnxuJ6X5OQf//KR+lXUlK2eYA3UXkCnBuoh6BtV1kd0eZ+4BFjq0zjztW\nvAXoBcx3bJy0V0Ruc5wjqCjupqYKNv2LRSkHY8wmEXkRmC0ifkASMAg4AzR1XDkcwhqHAGtt+88c\nX/zpl7ruBYwWkVcc5+iR1cu5750olTe67LZSlyAip4wxJb1dD6U8RbuSlLo0/etJFSl6xaCUUsqF\nXjEopZRyoYFBKaWUCw0MSimlXGhgUEop5UIDg1JKKRcaGJRSSrn4f+O7exYSL/ZXAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c3f8810>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.plot(train_loss, linewidth=3, label=\"train loss\")\n",
    "pyplot.plot(test_loss, linewidth=3, label=\"valid loss\")\n",
    "pyplot.grid()\n",
    "pyplot.legend()\n",
    "pyplot.xlabel(\"epoch\")\n",
    "pyplot.ylabel(\"loss\")\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mini Batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mini Batch allows a faster gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X_train_tf = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, X.shape[1]))\n",
    "    y_train_tf = tf.placeholder(tf.float32, shape=(batch_size, nb_classes))\n",
    "    X_test_tf = tf.constant(np.array(X_test).astype(np.float32))\n",
    "    y_test_tf = tf.constant((np.arange(nb_classes) == y_test[:,None]).astype(np.float32))\n",
    "    \n",
    "    weights = tf.Variable(\n",
    "        tf.truncated_normal([X.shape[1], nb_classes]))\n",
    "    biases = tf.Variable(tf.zeros([nb_classes]))\n",
    "\n",
    "    logits = tf.matmul(X_train_tf, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_train_tf))\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.8).minimize(loss)\n",
    "    \n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(X_test_tf, weights) + biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss at step 0: 4.617716\n",
      "Valid Loss: 4.640052\n",
      "Train Loss at step 4000: 3.103014\n",
      "Valid Loss: 3.245315\n",
      "Train Loss at step 8000: 2.139842\n",
      "Valid Loss: 2.453359\n",
      "Train Loss at step 12000: 1.759780\n",
      "Valid Loss: 1.968372\n",
      "Train Loss at step 16000: 1.318464\n",
      "Valid Loss: 1.650348\n",
      "Train Loss at step 20000: 1.119166\n",
      "Valid Loss: 1.427687\n",
      "Train Loss at step 24000: 0.949450\n",
      "Valid Loss: 1.267724\n",
      "Train Loss at step 28000: 0.779175\n",
      "Valid Loss: 1.146158\n",
      "Train Loss at step 32000: 0.690034\n",
      "Valid Loss: 1.052394\n",
      "Train Loss at step 36000: 0.688905\n",
      "Valid Loss: 0.978355\n",
      "Train Loss at step 40000: 0.530511\n",
      "Valid Loss: 0.917245\n",
      "Train Loss at step 44000: 0.473384\n",
      "Valid Loss: 0.867600\n",
      "Train Loss at step 48000: 0.526159\n",
      "Valid Loss: 0.825346\n",
      "Train Loss at step 52000: 0.430732\n",
      "Valid Loss: 0.789898\n",
      "Train Loss at step 56000: 0.343112\n",
      "Valid Loss: 0.759255\n",
      "Train Loss at step 60000: 0.403765\n",
      "Valid Loss: 0.732507\n",
      "Train Loss at step 64000: 0.368342\n",
      "Valid Loss: 0.709447\n",
      "Train Loss at step 68000: 0.287409\n",
      "Valid Loss: 0.688575\n"
     ]
    }
   ],
   "source": [
    "num_steps = 70001\n",
    "train_loss = []\n",
    "test_loss = []\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = (step * batch_size) % (len(y_train) - batch_size)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = np.array(X_train)[offset:(offset + batch_size), :]\n",
    "        batch_labels = y_train[offset:(offset + batch_size)]\n",
    "        batch_labels = (np.arange(nb_classes) == batch_labels[:,None]).astype(np.float32)\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {X_train_tf : batch_data, y_train_tf : batch_labels}\n",
    "        _, l, predictions = session.run(\n",
    "              [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 4000 == 0):\n",
    "            train_loss.append(l)\n",
    "            test_loss.append(log_loss(y_test, test_prediction.eval()))\n",
    "            print('Train Loss at step %d: %f' % (step, l))\n",
    "            print('Valid Loss: %f'% log_loss(y_test, test_prediction.eval()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEPCAYAAABBUX+lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Wl4FFX69/Hv6YQQkhDCGpYACasKyKKA4BZEVnXckBGV\nTUcHRtx1xAVx/jM6jo6oqA+KCoKjoCgoOiAoEBBhBJR934LsBEhICCEhyXlenE5nh0ro5YTcn+uq\nq7uqq7t+IeFU9V1V5yitNUIIIS58rkAHEEII4R/S4AshRCUhDb4QQlQS0uALIUQlIQ2+EEJUEtLg\nCyFEJRHs6w0opRKBE0AucEZr3cXX2xRCCFGczxt8TEMfr7VO9sO2hBBClMIfJR3lp+0IIYQ4C380\nxBr4QSm1Uil1vx+2J4QQogT+KOlcqbU+qJSqi2n4N2utl/phu0IIIQrweYOvtT7ofkxSSs0CugCF\nGnyllHToI4QQZaS1VmVZ36clHaVUmFIqwv08HOgNbChpXa21VdPYsWMDnkEyXTiZbM0lmSpupvLw\n9RF+NDDLfQQfDHyqtZ7v4216RWJiYqAjFCOZnLExE9iZSzI5Y2Om8vBpg6+13g108OU2hBBCOCOX\nS5Zi2LBhgY5QjGRyxsZMYGcuyeSMjZnKQ5W3FuTVEEppG3IIIURFoZRC23TStiJLSEgIdIRiJJMz\nNmYCO3OVlik2NhallEwWTLGxsV77ffvjOnwhRAWzZ8+ecl8JIrxLqTIdxJ/9s2z4pUpJRwi7uMsF\ngY4hKP13ISUdIYQQpZIGvxQVqd4aSJLJORtz2ZhJ+I40+EKISmfkyJG89NJL5Xpvjx49mDRpkpcT\n+YfU8IUQxdhcw4+Li+Ojjz7iuuuuC8j2e/ToweDBg7n33nv9sj2p4QshRClycnICHcFa0uCXwsba\npmRyxsZMYGcuGzOdzZAhQ/j999+56aabiIyM5N///jd79uzB5XIxadIkmjZtSs+ePQEYOHAgDRo0\noGbNmsTHx7Np0ybP5wwfPpwXXngBgMWLF9O4cWPGjRtHdHQ0jRo14uOPP3aUR2vNP/7xD2JjY6lf\nvz7Dhg0jNTUVgMzMTAYPHkydOnWoWbMmXbt2JSkpCYCPP/6Y5s2bExkZSfPmzZk2bZoX/5VKZ02D\nfzDtYKAjCCEcUMq7U1lMnTqVJk2a8N1335GamsqTTz7peW3JkiVs2bKFefPmAdC/f3927tzJkSNH\n6NSpE3fffXepn3vo0CHS0tI4cOAAH374IQ8++CAnTpw4Z57JkyczdepUFi9ezK5du0hLS+Ohhx4C\nYMqUKaSmprJ//36OHz/Oe++9R7Vq1Th16hSPPPII8+bNIzU1lWXLltGhg5+6HAt0F5/u2pR+fdGH\nWghhB9M0lPaad6eyio2N1QsWLPDMJyYmapfLpRMTE0t9T3JyslZK6dTUVK211sOGDdNjxozRWmud\nkJCgw8LCdE5Ojmf9evXq6V9++aXEz4qPj9cfffSR1lrrnj176gkTJnhe27p1qw4JCdE5OTl60qRJ\n+sorr9Tr1q0r9P709HRds2ZNPXPmTJ2RkXHOn7e034V7eZnaWmuO8D9d+W2gIwghKrCYmBjP89zc\nXEaPHk2LFi2IiooiLi4OpRRHjx4t8b21a9fG5cpvDsPCwjh58uQ5t3ngwAGaNm3qmW/atClnzpzh\n8OHDDB48mD59+nDnnXcSExPD6NGjycnJISwsjM8//5wJEybQoEEDbrrpJrZu3XoeP7lz1jT4a0/+\nwOns04GO4WFjbVMyOWNjJrAzV3kyefsYv6xK62qg4PLPPvuMb7/9loULF5KSkkJiYuJ5DRxSmoYN\nG7Jnzx7P/J49e6hSpQrR0dEEBwczZswYNm7cyLJly/j222+ZOnUqAL169WL+/PkcOnSI1q1bc//9\n/hnu25oGP8d1igW7FgU6hhDCcvXr12fXrl2FlhVtyNPS0qhatSo1a9YkPT2dZ555xqt90uQZNGgQ\nb7zxBomJiZw8eZLnnnuOO++8E5fLRUJCAhs2bCA3N5eIiAiqVKmCy+XiyJEjzJ49m1OnTlGlShUi\nIiIICgryeraSWNPgA0xd/l2gI3jEx8cHOkIxkskZGzOBnblszHQuo0eP5u9//zu1atVi3LhxQPGj\n/iFDhtCkSRMaNWpE27Zt6d69e5m2cbadQ8HX7r33XgYPHsw111xD8+bNCQsLY/z48YA5ETxgwABq\n1KhBmzZtPNfv5+bmMm7cOBo1akSdOnVYsmQJEyZMKFO+8rLmxitehBqqMclj9vhkTyyEcM7mG68q\nmwv2xqsTei/rj6wPdAzgwqm3+ppkcs7GXDZmEr5jVYMP8NUGuVpHCCF8waqSDsBF4Vew+cnlAc0j\nRGUnJR17XJglnVwTZcvJXziSfiTAYYQQ4sJjT4O/90rzqDRzts8JbBbsrG1KJmdszAR25rIxk/Ad\naxr8oJ03eZ5/sdaeyzOFEOJCYU0Nv9sfNrG80yUAVFURnHj2KFWDqwY4mRCVk9Tw7XFB1vBvveoi\nON4MgEx9kiV7lgQ4kRBCXFisafD79FGwLb+sM3trYC/PtLG2KZmcsTET2JnLxky+ktfvfZ62bduy\nZEnJB5ZF1y3K5XIV696hIrCmwW/XDmom3eiZn7nhO/lKKYTwqoJ38W/YsIFrrrnG0bplec1m1jT4\nSkH/NtdAZnUADmTsZlPSpnO8y3ds7GNEMjljYyawM5eNmSqCinowak2DD9C/Twjs6OOZ/26bXK0j\nhMj36quvcscddxRa9sgjj/Doo48CZujASy65hMjISFq0aMHEiRNL/ay4uDgWLlwIwOnTpxk2bBi1\natWibdu2rFy50nGm1NRUhgwZQr169YiLi+Oll17yvLZz507i4+OJioqiXr16DBo0yPPaY489RnR0\nNDVq1KB9+/aFhmD0lWCfb6EMrr8eePUmaPMlAF9v/panr3o6IFkSEhKsO/qRTM7YmAnszFWeTOpv\n3i1n6LHOj5bvvPNO/u///o/09HTCw8PJzc1lxowZfPPNNwBER0czZ84cYmNj+emnn+jbty9dunQ5\n5xCCL774Irt372b37t2cPHmSvn37Os40atQo0tLSSExMJCkpid69e9OwYUOGDx/OmDFj6NOnDwkJ\nCWRlZbFq1SoA5s+fz9KlS9mxYwfVq1dn69atREVFOd5meVl1hF+vHrSr1g/cVxqtOLCco6dKHqFG\nCFH5NGnShE6dOjFr1iwAFixYQHh4OJ07dwagX79+xMbGAnD11VfTu3dvfvrpp3N+7owZM3j++eep\nUaMGjRo14uGHH3aUJzc3l88//5xXXnmFsLAwmjZtyhNPPMEnn3wCQJUqVdizZw/79+8nJCTE001z\nlSpVSEtLY9OmTWitad26NdHR0WX95ygzqxp8gBvi68LebgDkksvc7XMDksO2IzGQTE7ZmAnszGVj\npnMZNGgQ06ZNA2DatGncddddntfmzp1Lt27dqF27NjVr1mTu3LmlDmtY0IEDBwoNkVhw2MKzOXr0\nKNnZ2TRp0qTQe/fv3w+YElRubi5dunShXbt2TJ48GYAePXowatQoHnzwQaKjoxkxYoSjIRXPW1kH\nwfXFRIFBehcu1JqrXta8iOZF9MAZA885yK8Qwrsoz+jifpKUlKTDwsL0vn37dFRUlN6yZYvWWuvM\nzEwdFhamZ86c6RmQ/JZbbik0WHnjxo09n1NwMPS4uDg9b948z2sTJ04stG5RSim9c+dOnZOTo6tW\nrao3b97see3999/XPXr0KPaepUuX6tDQUL1z585iP098fLx+4YUXStxWab8LKvIg5nm6d4dqe/Ov\nx5+z7XuycrL8nsPG65MlkzM2ZgI7c9mY6Vzq1KnDtddey/Dhw2nWrBmtW7cGICsri6ysLOrUqYPL\n5WLu3LnMnz/f0WcOHDiQf/7zn6SkpLBv3z7eeecdR+9zuVwMHDiQ5557jpMnT7Jnzx7eeOMNBg8e\nDMCXX37pOdqPiorC5XLhcrlYtWoVK1asIDs7m2rVqhEaGlpoEHVfsa7Br1oVerRtAynmK9XJM6ks\n/X1pgFMJIWxy1113sWDBAu6++27PsoiICMaPH88dd9xBrVq1mD59OjfffHOpn1HwWvqxY8fSpEkT\n4uLi6Nu3L0OGDDnr9gu+d/z48YSFhdGsWTOuueYa7rnnHoYPHw7AypUr6dq1K5GRkdxyyy2MHz+e\n2NhYUlNTuf/++6lVqxZxcXHUqVOHp556qrz/HI75pS8dpZQLWAXs01r/oYTXdcEcb78ND899CLqa\nveyjXR/ljb5v+DynEMKQvnTsURH70nkEcHyRae/ewLb8u26/3fat/PEJIcR58nmDr5SKAfoDHzp9\nT6tW0CQ3HrLCAdiZvJOtx7b6JmApbKxtSiZnbMwEduayMZPwHX8c4b8BPAU4PkRXCvpcXxV29vYs\nk7tuhRDi/Pj0Tlul1A3AYa31GqVUPFBqvWnYsGGeGyaioqJo1KgDrLwJLp4Fu+GT2Z/wZPcngfyj\nkrxriH01n8df26uI8/Hx8VblyVPwDtJA57F5/my/P2GPvN9JQkICiYmJ5f4cn560VUq9DNwDZAPV\ngOrATK31kCLr6aI5kpOhdtPD6McbgNIEqSCOPHWEWtVq+SyvEMKQk7b2qDAnbbXWz2qtm2itmwF3\nAguLNvalqVkTuraJhv1dAMjROXy/43sfpi3MxqMcyeSMjZnAzlw2ZhK+Y911+AX16UOhq3Wkji+E\nfzRt2hSllEwWTE67eXDCmjFtS8qxbBlcedtaGGl6uosKjeLIk0eoElTF3xGFEMIq1pV0zleXLhB5\n+lI4YYYaSzmdwrK9ywKcSgghKiarG/zgYLi+pyp2E5Y/2FjblEzO2JgJ7MwlmZyxMVN5WN3gQ/G7\nbqWOL4QQ5WN1DR9g925o1uo0/LU2hJwCYNuobbSs3dKfEYUQwioXXA0fIC4OWsaFwq7rPcvkKF8I\nIcrO+gYfSrg8c7vvG3wba3aSyRkbM4GduSSTMzZmKo8K0eD37g1sv8Ezv2TPEk6cPhG4QEIIUQFZ\nX8MHOHkSatWCM8Mvh4a/AvD5gM8Z2GagvyIKIYRVLsgaPkBEhBn6kK35Qx/66/JMIYS4UFSIBh+K\nX545Z/sccnJzfLY9G2t2kskZGzOBnbkkkzM2ZiqPCtPg9+kDHOwEqQ0BOJ5xnOX7lgc2lBBCVCAV\nooYPkJsL0dFwtNsDcNkHADx95dO8cv0r/ogohBBWuWBr+AAuF/TqhdTxhRCinCpMgw/uOv7unnAm\nFIBNSZvYlbzLJ9uysWYnmZyxMRPYmUsyOWNjpvKoeA3+mTDT6LvJXbdCCOFMhanh52nXDjaEvgc3\njgSgV7NezB8835fxhBDCOhd0DT9P0cszExITSM1MDVwgIYSoICpcg9+nD5AaAwfNKFhncs/ww84f\nvL4dG2t2kskZGzOBnbkkkzM2ZiqPCtfgX301hIYC2+RqHSGEKIsKV8MHc5Q/f+MKuL8rAHXD6nLw\niYMEuYJ8FVEIIaxSKWr44K7jH7gcTkYDkHQqiRX7VwQ2lBBCWK5CNvh9+gDaBdvyu0z29uWZNtbs\nJJMzNmYCO3NJJmdszFQeFbLBb9MGGjRA6vhCCFEGFbKGDzBsGEyZdtKMdRucBUDiI4k0jWrqg4RC\nCGGXSlPDB3dZJysCdl/nWSZ33QohROkqbIN//fWgFD4b69bGmp1kcsbGTGBnLsnkjI2ZyqPCNvh1\n60KnThRq8BfuXsjJrJOBCyWEEBarsDV8gGefhX/+Exh5KUSvB2DWH2dxy0W3eDmhEELYpVLV8MFd\nx4dCfeRP2zAtMGGEEMJyFbrB79YNwsOBzbd7ls3YOIN1h9ed92fbWLOTTM7YmAnszCWZnLExU3lU\n6AY/JAR69MCMdbvV1PI1mucXPh/YYEIIYaEKXcMHeOcdeOghIHodjOgAynzOz/f+TPfG3b2YUggh\n7FHpavjg7lcH4PClBG8e5Fn+7IJnsWFnJoQQtqjwDX7LlhAba55n//g3glQwAIv3LOaHXeXvJ9/G\nmp1kcsbGTGBnLsnkjI2ZyqPCN/hKFTjKP96CS7Pv87wmR/lCCJHPpzV8pVRVYAkQAgQDX2qt/1bC\neuWu4QPMmgW33WaeN2i1n+QhLTidfRqAGXfMYMAlA8r92UIIYSPravha60ygh9a6I9AB6KeU6uLt\n7fTvD/XqmecHtzWiV+RDntfGLBpDdm62tzcphBAVjs9LOlrrU+6nVTFH+V7/SlG1KowYkT9/6Kun\niawaCcCWo1v4ZO0nZf5MG2t2kskZGzOBnbkkkzM2ZioPnzf4SimXUmo1cAj4QWu90hfbGTECqlQx\nz1curs2gpk96Xntx8YtkZmf6YrNCCFFh+O06fKVUJPA1MEprvanIa+dVw89zzz3w6afm+Z1D0ljQ\ntjlJp5IAeKvvWzzc9eHz3oYQQtigPDV8v954pZQaA6RrrccVWa6HDh1KrPv6yqioKDp06EB8fDyQ\n/3XqXPPh4fF06QKQQFAQjJ27lheWPQq7oUZoDfaN30dESITjz5N5mZd5mbdlPu95YmIiAFOmTClz\ng4/W2mcTUAeo4X5eDXPFTv8S1tPe0q2b1mCm58Zm6MbjGmteRPMi+u+L/+74cxYtWuS1TN4imZyx\nMZPWduaSTM7YmMndbpapTfZ1Db8BsEgptQb4BZintZ7jyw0+XKBq88GEUJ678kXP/GvLXuN4xnFf\nbl4IIaxV4fvSKerMGYiLg/37zfzkKdm8ktqWrce2AvDX7n/lX73+5ZVtCSFEoFh3HX4gVKkCf/lL\n/vw744P5vx5/98yPXzGeA2kHApBMCCEC64Jr8AEeeABCQ83zX3+FBsm306lBJwBOZ5/mH0v+cc7P\nKHiixBaSyRkbM4GduSSTMzZmKo8LssGvUwfuvjt//u3xLl6+7mXP/Ae/fcDO4zsDkEwIIQLngqvh\n51m3Dtq3N8+DgmDXLs3ghfEs2bMEgLvb3c1/bvuPV7cphBD+IjX8Ai69FNyXsZKTAxMmKP7Z85+e\n1z9b/xnrD68PTDghhAiAC7bBB3jkkfznEydCh9rdubFVgaEQF5U+FKKNNTvJ5IyNmcDOXJLJGRsz\nlccF3eDfdFP+4CjHj5tuF1667iXP67O3zmb53uWBCSeEEH52wdbw87z+Ojzp7ketbVtT27975l1M\n2zANgPjYeBYOWYhSZbtDWQghAsn6vnRKDeHDBj8lBWJiID3dzC9YAE067ODidy/29JM//5759Gre\nyyfbF0IIX5CTtiWIioKhQ/Pn33oLWtRqwX0dCwyFuLD4UIg21uwkkzM2ZgI7c0kmZ2zMVB4XfIMP\n8FD+AFh8+y3s2gVjrhlDaLC5O2vVgVXM3DwzQOmEEMI/HJV0lFKPAJOBNOBDoCMwWms93yshfFjS\nydO3L8ybZ54/9hiMGwdPzX+Kfy//NwAX1bmIDSM3EOQK8mkOIYTwBl+WdO7VWqcCvYGawGDglTLm\nC6iCl2h+9BGkpcHoq0ZTPaQ64B4KcV3Zh0IUQoiKwmmDn7cX6Q98orXeWGBZhdCnD7RqZZ6npsKU\nKVA7rDZPds8fCnFswljPUIg21uwkkzM2ZgI7c0kmZ2zMVB5OG/xflVLzMQ3+PKVUdSDXd7G8z+Uq\n3Ff+229Dbi48dsVj1A2rC8DvJ35n4q8TA5RQCCF8y2kN3wV0AHZprVOUUrWAGK31Oq+E8EMNH0wZ\nJybGHOEDzJkD/frBm/97k8fmPQZAvfB67Hx4JxEhET7PI4QQ5eXLGn43YKu7sb8HeB44UdaAgVa9\nOtyXfzUmb71lHkdcPoLGkY0BOJJ+hLf+91YA0gkhhG85bfAnAKeUUu2BJ4CdwFSfpfKhUaMg76ba\nefNgyxYIDQ5l7LVjPeu8tuw1Zs+bHaCEpbOxjiiZnLMxl2RyxsZM5eG0wc9211xuBt7RWr8LVPdd\nLN9p1gz+8If8+bffNo9DOwylde3WAJzIPMH0DdMDkE4IIXzHaQ1/MfA9cC9wNXAEWKu1bueVEH6q\n4edZtAiuu848Dw+HffvMHbkzNs5g4JcDAagWXI3Vf15N6zqt/ZZLCCGc8mUN/49AJuZ6/ENADPBa\nGfNZIz7edKQGpo+djz4yz2+/JH8oxIzsDPp/1p+k9KTAhBRCCC9z1OC7G/lPgRpKqRuB01rrClnD\nB1PDL3gj1jvvmEFSXMrFBzd9QFiVMNgNu5J3ccvnt3A6+3TgwhZgYx1RMjlnYy7J5IyNmcrDUYOv\nlBoIrADuAAYCvyilBvgymK/dfTfUrm2eJyaaPnYAOjXoxGe3feZZb9neZQz7ehi5ukLddiCEEMU4\nreGvBXpprY+45+sCP2qt23slhJ9r+HmefRb+6R71MD7e1PbzFLw2H+C5q5/jH9f9w78BhRCiFL6s\n4bvyGnu3Y2V4r7VGjjQDnAMkJJjBUfI80vURHuz8oGf+pZ9eYvLqyf4NKIQQXuS00f5eKTVPKTVM\nKTUM+C8wx3ex/KNxY7j99vz5twrcb7V48WLe7Psm/Vr08yx74LsHWLh7oR8TFmZjHVEyOWdjLsnk\njI2ZysPpSdungInApe5potb6aV8G85eCJ28//RSOHs2fD3YF8/mAz7k0+lIAsnOzue3z29ictNnP\nKYUQ4vxd8EMcnovW0KULrFpl5l96ydT2C9qXuo+uH3blQNoBAGKjYvnfff8jOiLaz2mFEMLweg1f\nKZWmlEotYUpTSqWeX1w7KFW4F83/9//gzJnC68RExvDtoG8JrxIOQGJKIjdPv5mMMxl+TCqEEOfn\nrA2+1rq61jqyhKm61jrSXyF9beBAiHYfrO/fDzNnFq/ZdWrQiekDpuNS5p/sl/2/MPTroX69XNPG\nOqJkcs7GXJLJGRszlUeFv9LGG6pWNVfs5HmrlM4yb2x1I2/2edMzP2PTDJ5b8JyP0wkhhHdU+hp+\nnsOHzVU7eeWcFSugc+eS131k7iOMXzHeM//BTR/wp05/8kNKIYQwfHkd/gUvOhruvDN/vrSjfIBx\nfcZxU6ubPPMjvhvBDzt/8GE6IYQ4f9LgF1DwEs3p0xNYvrzk9YJcQXx2+2d0rN8RgBydw4AZA9h4\nZKNP89lYR5RMztmYSzI5Y2Om8pAGv4DLLoOrrjLPc3KgZ0+YO7fkdSNCIvjuru+IiYwBIDUzlRs+\nu4FDJw/5Ka0QQpSN1PCL2LDB9JWf5O4VOTgYJk+Ge+4pef21h9Zy1eSrOJl1EoDODTuTMCzB9Lgp\nhBA+Yl0NXykVo5RaqJTaqJRar5R6+NzvCqy2beHnnyE21sxnZ8PgwTBuXMnrt6/fni8GfOG5XHPl\ngZXcM/Me6V1TCGEdX5d0soHHtdZtMAOhP6iUusjH2zxvLVvCa68l0K7AeF5PPAGjR5s7c4vq17If\n7/R7xzM/a8ssnv7B+z1P2FhHlEzO2ZhLMjljY6by8GmDr7U+pLVe435+EtgMNPLlNr2lTh1YsiS/\npg/wr3/Bn/5kjvqLGtl5JI9f8bhn/t/L/817q97zQ1IhhHDGbzV8pVQskAC0dTf+BV+zpoZfVEYG\n/PGP+QOkgBkEffp0qFat8Lo5uTnc/sXtfLP1GwCCVBDf3fUdfVv09WNiIURlYF0NP49SKgL4Enik\naGNvu2rVTFcLw4fnL5s9G3r3hpSUwusGuYL49LZPuazBZYC5XHPgjIGsPbTWj4mFEKJkPj/CV0oF\nA98Bc7XWJd7OpJTSQ4cOJdZ9pjQqKooOHToQHx8P5NfP/Dm/Zs0aHn30Uc+81jBvXjz/+heYLyrQ\nrl08338P27YVfv9Xc75i5H9HklTPXOoTui+Uv3b/K38b/rfzype3LBD/HqXNF80W6DwAb775ZsD/\nfkqaz1tmSx75/TmfL9oeBCJP3vPExEQApkyZUuYjfLTWPp2AqcC4c6yjbbNo0aISl48bp7U5dWum\n2Fitt24tvt66Q+t05D8jNS/imR6a85DOzM70eqZAkkzO2ZhLMjljYyZ3u1mm9tinR/hKqSuBJcB6\nQLunZ7XW3xdZT/syh7f95z+mxJN38rZuXXOD1mWXFV5vzaE1DPhiADuTd3qWdWnUhS8GfEHTqKZ+\nTCyEuNCUp4YvN16V09y5ZnjEDHeX+BER8PXX5u7cgk6cPsHwb4Yza8ssz7Ja1Wrxn1v/Q7+W/RBC\niPKw9qRtRVSwblaSfv1gwQKoWdPMnzwJ/fvDjBmF16sRWoOvBn7F671fJ9gVDMDxjOP0/6w/zy14\njuzcEq7xLGemQJBMztmYSzI5Y2Om8pAG/zx06wZLl0KM6U6HrCxzCeeECYXXU0rxeLfHWTxsMY2q\n59+G8PLSl+n9SW/pf0cI4RdS0vGC33+HPn1gy5b8ZWPHmkkV+cKVlJ7EPbPuYf7O+Z5l9SPq8/mA\nz7mm6TV+SiyEqOikpBMgTZrATz9B1675y/72N3jwQdPrZkF1w+sy5645vHjtiyjM7+rQyUNcN+U6\n/rX0X9IHjxDCZ6TBL0VZa3Z16sCPP5oj/TwTJsCgQZCZWXjdIFcQY+PHMu+eedQJqwOYm7RGLxjN\nzdNvJjkj2SuZ/EEyOWdjLsnkjI2ZykMafC+KiDB34Q4alL9sxgy45RZT3y+qV/NerP7zaro37u5Z\n9t227+g0sROrDqzyQ2IhRGUiNXwfyM2Fxx8vPEzigAEwbZrpX7+oMzlneGbBM7y+/HXPspCgEN7s\n8yYjLh+BKnoiQAhR6cl1+BbR2py0/fvf85cNGwYffQSuUr5Xzdo8i2HfDCM1M9WzbFDbQUy8aSIR\nIRG+DSyEqFDkpK0XnW/NTilz4tbd/QYAH38Mjz1Wcp/6ALdefCu/PfCbZ6xcgGkbptH5g85sPLLR\nyjqiZHLOxlySyRkbM5WHNPg+pJQZKeu++/KXjR8PL7xQ+nua12rOsvuW8UCnBzzLthzdQpcPuzB/\nx3wutG9CQgj/kZKOH+TkwF13wRdf5C979VV46qmzv++TtZ8w4r8jOHXmlGfZ1U2u5pXrXyl0olcI\nUflIDd9iWVlw660wZ07+svfegz//+ezv23hkI7d/cTtbj20ttPwPrf/Ay9e9TJt6bXyQVghhO6nh\ne5G3a3ZUCpmbAAAah0lEQVQhIfDll3DttfnLRo6ETz89+/va1GvDyvtXMqrzKIL2BHmWz946m0vf\nu5Th3wzn9xO/ezVrWdhY27QxE9iZSzI5Y2Om8pAG34+qVTNDJXbpYua1hqFD4Ztvzv6+6lWr83b/\nt5l661TuaneXZ3muzuXjNR/T8u2WPD7vcY6eOurD9EKIik5KOgFw/Lg50t+wwcyHhMB//wvXX+/s\n/WsOreGZBc/w/Y5CwwoQWTWSp7o/xaNXPCqXcQpxgZMafgVy8CBccw3s2GHmw8Lghx+gexnOxSYk\nJjD6x9H8sv+XQsujw6MZc80Y7r/sfkKCQryYWghhC6nhe5Gva3YNGpi+d/K6Vj51yvSnv2aN80zx\nsfEsv285MwfO5KI6F3mWH04/zKi5o7j43YuZtn6aTztks7G2aWMmsDOXZHLGxkzlIQ1+ADVtahr9\nunXN/IkT0Lt34W6Wz0Upxa0X38r6kev58KYPC/W3vyt5F3fNvIvLJ17OvB3z5Bp+ISo5KelYYO1a\niI+HlBQzHxNjuluOjS37Z2WcyeDdle/y8k8vk3y6cK+b8bHxvNLzFbrGdC3l3UKIikJq+BXY8uXQ\nqxekp5v55s1No9+gQfk+L+V0Cq/+/Cpv/u9NMrIzCr1228W38fzVz9OxQcdS3i2EsJ3U8L3I3zW7\nbt3M5Zkh7nOsO3eaHcCxY+XLFBUaxcs9X2bHwzv482V/JkjlX8M/c/NMOk3sRKf3O/HuindL7X/f\nCRtrmzZmAjtzSSZnbMxUHtLgW6RnT9N/fpC7bd640QyWnpp69vedTcPqDXnvxvfY9OAmBrYZWOi1\n1YdWM2ruKBq83oBBXw3ih50/yIhbQlzApKRjoc8+g3vuye9V85prYO5cc+nm+fr1wK+8vvx1Zm6e\nSWZOZrHXm9RowvAOwxnWYRixUbHnv0EhhE9IDf8C8v77MGJE/ny/fvD11/kln/OVnJHMtA3TmLR6\nEr8e/LXEdXrG9eS+jvdx68W3Ehoc6p0NCyG8Qmr4XhTomt2f/wyvvZY/P3cu9OqVwIkT3vn8mtVq\n8pfOf2HVA6tY8+c1PNzlYWpVq1VonQW7F3DXzLto8HoDRs0ZxW8Hfyt2aWeg/51KYmMmsDOXZHLG\nxkzlIQ2+xZ58Ep5/Pn9+yRJz7f6YMXDUi93mtK/fnrf6vcWBxw/wxYAv6NuiL4r8A4eU0ym8u/Jd\nLpt4GR3f78j4X8Zz7NSxs3yiEMJGUtKxnNZm1Kzx4wsvDwsz3wKeeAIaNSr5vedj74m9TFk7hUmr\nJ7E7ZXex10OCQrjlolsY1n4YPZv1lC4chPAzqeFfoLSGTz6Bl16CbdsKv1alihkr9+mnzbX73par\nc1mcuJhJaybx5aYvOZ19utg6ESER9GrWixta3kC/lv1oWL2h94MIIQqRBt+LEhISiI+PD3SMQhYs\nSODYsXheftncnVuQywV33gnPPANt2/pm+ymnU5i+YTqTVk9i5YGVZuFuIK7weh3rd+SGljdwQ6sb\n6NywM0GuoGKf5Us2/u7AzlySyRkbM8lJ2wtcUBAMHAirV8N335mbtfLk5prLOdu1g1tugZUrvb/9\nqNAoRlw+ghX3r2DdiHU8dsVj1I+oX2y91YdW84+f/kG3j7pR//X6DJ41mOkbpp/XDV5CiPMnR/gV\nmNbmRO5LL5mulYu6/np47jnT974q03FAWTJothzdwpztc/jv9v/y0+8/kZ2bXeK6LuWie+Pu5ui/\n5Q20rdcW5atgQlzgpKRTia1cCS+/bK7VL6pbN9Pw9+/vu4Y/z4nTJ/hh1w/M2T6HOdvncDj9cKnr\nNo5sTP+W/bmh5Q1cF3cd4SHhvg0nxAVEGnwvsrFm5yTTxo3wyiswbRrk5BR+rX17U+MfMCC/+wZf\nZsrVufx28DfP0f/K/SvRlPx7rhpUle6Nu3umK2KuKHZfgDcyBZKNuSSTMzZmKk+DH+yrMCIw2rQx\nV/T87W/w6qsweTJkZZnX1q41J3ZbtoTHH4e774bq1X2XxaVcXN7wci5veDkvXPsCR9KP8P2O7/nv\n9v8yb8c8TmTm30WWmZPJosRFLEpc5Fl2UZ2L6B7TnW6Nu9G9cXcuqnMRLiWnnYQoLznCv8Dt3w/j\nxsF775lRtQqKiDB99owYYY7+/elMzhmW7V3mOfrfmLTxnO+JCo3iipgrPDuBro26Ur2qD/dYQlhM\nSjqiVEePmpu33n47f6CVgrp1Mw3/HXdAtWr+z7cvdR/L9i5j2d5lLN+3nN8O/lbqyd88LuWibb22\ndI8xZaBujbvRvGZzOREsKgXrGnyl1EfAjcBhrfWlZ1nPugbfxpqdNzKlpsKUKTBhAmzeXPz1mjXN\njVwjRkCrVv7JVJKMMxmsOrCK5fuWe3YCR9KPnPN9dcPq0jKtJdf3uJ520e24NPpSmtds7vd7AUpy\nof5NeZtkcsbGGv5k4G1gqo+3IxyKjISHHoJRo8yIWu+9B19+CWfOmNeTk+GNN8x03XWm4b/5Zu/1\n0ulUtSrVuLrp1Vzd9GrAXP65K3mXp/FftncZ64+sL9Z/f9KpJJJ+T2LZkmX5nxVcjTb12tCuntkB\n5D3WDa/r159JiEDzeUlHKdUU+LaiHeFXJkeOmJO7778Pu4t3m0N0NPzpT3D//abzNlukZaaxYv8K\nz05g+b7lpJwuoV5Viujw6EI7gHbR7bik7iXSFbSoEKwr6YA0+BVJbq65geu992D2bDNfkFLmWv6R\nI6FvX+9d2uktuTqXLUe3sOrAKtYfXs+6I+tYf3g9B08edPwZLuWiVe1Wnp1Am7ptaFW7Fc1rNZcd\ngbBKhW7whw4dSmxsLABRUVF06NDBUzPL64van/Nr1qzh0UcfDdj2S5rPW+aP7SUlwcaN8XzwARw4\nkLf9ePdjAvXqwcMPx3PxxQnUqsU5P8+f82+++Wahv59vvv+GXcm7cMW5WH9kPUuXLCUxJZHMxu4R\nv/K+1eT1CVTKfNMOTWlZuyXh+8KJqRFD/179aVmrJXvW7CE4KNiq35/T+aLZAp0Hiv/+Ap3HlvYg\n73liYiIAU6ZMqbgNvm1H+AkWnqQJRKbsbPj2W3PUP39+8dddrgRuuy2ekSOhRw/f38nrhJN/p5zc\nHHYl72L9kfWFvg3sOL6j1JvDShPsCqZZzWa0rNWSVrVb5T/WbklMZIzn3gH5m3JGMjlj6xF+LKbB\nb3eWdaxr8EVxO3bAxIkwaRIcK2H8k9atzUneoUPN1T4VUXpWOpuSNrHu8DrWH1nP1mNb2XZsG4kp\nieUa4D00OJQWtVrQqnYrYmvE0jSqKU1qNKFpDfNYq1otuYxUlIt1Db5S6jNMHaA2cBgYq7WeXMJ6\n0uBXIJmZ8NVX5tLOpUuLvx4aau7oHTECunSx46j/fGVmZ7I7ZTfbjm1j+7Ht5vG4edyftr/cnxte\nJdzsAKKa0iSySbEdQqPIRgS75IZ4UZx1Db7jEBY2+DZ+hbMx0+TJCfz6azxTp0JaWvHXO3Y0J3kH\nDTJ39vqDv/+d0rPS2XF8R6GdQN7j0VMFxqIsYeyAc3EpF42qNyq0I2gc2ZhGkY1oWL0hDas3pF54\nvXLvFGz8m5JMzth4Hb64wMXFwfDh+R22TZhg+uvPs3o1PPCAGZ938GBz1O+rAVoCJTwknPb129O+\nfvH+KZIzktl+fDs7ju9g0aJFBDUL4vcTv7PnxB72pOwh/Uz6WT87V+eyN3Uve1P3lrqOS7mIDo/2\n7ABKm+qE1ZG+iCo5OcIXXqW16ap5wgSYPh1OFx8RkauuMkf9t98OVav6P6MttNYkn042O4CUPZ4d\nQcEdwtm6ly6rYFcwDSIaFNoJRIdHUy+8HvXC61E3vK7neY2qNeTcguWkpCOscvw4TJ1qrvDZurX4\n63XqwL33msHYmzXzf76K4HT2afae2FtoZ7D3xF4OnjzIgbQDHEg7QNKpJK9vt4qrSqEdQL3wetQN\nK31exjLwP2nwvcjGml1FzaQ1LFpkGv5Zs8ylnkX16WPq/FdcYbpvdp1H5cHGfyfwXa6snCwOnTzk\n2QEUnfJ2Dsczjhd/cznOK5SkWnA1aofVpna12p7HWtVqFZovtDysNjVDa5bYx5GNvz8bM0kNX1hJ\nKdMvz3XXwcGD5rLOiRPh99/z15k3z0wAUVHm6p6uXfMf60q3N6UKCQqhSY0mNKnR5KzrZZzJKLZj\nWPnzSqq1rMaRU0c4km6mpPQk0rJKOAN/ts/OzmBf6j72pe5z/B6FIio0qtiO4NS2UySQQFRoFDVD\na5rHajULzUeEREjJqRzkCF8ERE4OzJljjvrnzjXfAs4mLq7wDqBjx8B041xZZJzJIOlUUqGdQN7z\nojuHI+lHyMzJ9Gu+IBVEVGhUiTuDgjuJyKqRRFaNpEbVGp7neVOVoCp+zextUtIRFdLu3fDZZ7Bs\nGfzyS8k3dRUVHGwGbcnbAXTtarpzPp9SkCgfrTVpWWkczzjOsVPHOJZxjGOnjpl59/NjGcWXl6Wj\nO18IDQ4tdWdQdEdRvWp1qodUL/QYERJB9RDzGIjut6XB9yIba3aVIZPWsGsXrFhhGv9ffjGXdmY6\nOICsUQM6d4batRNo3Trec8OXUpz1+dlea9oUbrwRwsLO/2erDL+/ssjOzSY5I5ljGccK7SxW/ryS\n2pfUJuV0CimnU0g+nWweM5I986fOnDr3BrzpHOc6wqqEFdsReHYOBZ5HhERQvWp1+rboS2xU7HlF\nkhq+qPCUgubNzTRokFmWlQXr1uXvAH75BbZtK/7eEyfgxx+9nykyEv74R3NFUdeuF8adwzYIdgVT\nN7xusXEJYlNiz7kTysrJ4sTpEyXuDArOp2alkpppphOnT3iep2amkqNzvPaznDpzilNnTjm+jPab\nO7857wa/POQIX1RIycnmev+CO4GjR8/9vvN10UXmRrPBg6FBA99vT/iG1pqM7IwSdwSpmamcyCy+\nLC0rjbTMNNKy0jiZdbLQ87JaOGQhPeJ6nNfPICUdUWlpDYmJ+Uf/eX35a51/QjjvedH50l7LyjIn\nlLdvL769oCDo1880/jfe6P8RwYQ9cnUu6VnpxXYEBR9PZp0stOzpK5+mea3m57Xd8jT4aK0DPpkY\ndlm0aFGgIxQjmZzxZqbcXK2XLtX63nu1jogouFvIn+rU0frRR7Veu9Z/ubxFMjljYyZ3u1mmtlau\naRDiLJSCK6+Ejz4y9xB8/DFce23hdY4ehTffNFcNXX45vPuuuctYCNtISUeIcti50zT+U6bA3hL6\nNQsJgVtvNSWf66+3bzhIUfFJDV8IP8vJgQULzCDws2aVfPloTAwMGWK+GbRoAU2amPsIAik9Hfbt\ng3r1Ku5gNZWdNPheJNdMOyOZ8iUnmy6iJ0+GVatKWiMBiCc42Fzfn3f5acGpWTMI90I/ZKdPm5PY\nedPu3YUfkzz9rSVwySXxdO9uSldXXml2SoG89FT+ppyR6/CFCKCaNeEvfzHT+vWm4f/Pfwo2rkZ2\ntikJ7dxZ8ufUr28a/xYtiu8Qatc2jXFWlumLqKTGfPduOHTIee5Nm8z04Ydmvm5d6N4dz07gssvM\nKGai4pMjfCF8KCvL9Bk0e7a5vHPnTnPyt7wiI6F6dThw4Nz9D51NlSrQsCHs319y76UFhYSYRv/K\nK/N3BNHR5d+28A4p6QhRAZw6ZbqPyDvKLzglJp67AXYiKAgaNzadzsXGFn9s0MCsc+qUuYFt2TL4\n+WfzmJx87s9v0SL/G0D37nDJJdKPkb9Jg+9FNtbsJJMzNmYCZ7mys81VPyXtDHbuhJPumzqVMieD\nCzbiBZ/HxDg7MVw0U26uGazm55/zdwAldWNRVI0apiO7vKlz5/LfiWzj78/GTFLDF6KCCw42jXZc\nnLmcsyCtzfmAkydNg+6Lu3tdLrj4YjP96U9mWVKSafjzvgWsWlX8aqQTJ+CHH8yUJyam8E7gsstM\nScoXzpwx5y62bzc7qO3bzTmO6tXNlUh165rHvClvPjKycvWNJEf4QogyycyE337L/xbw88/FT0yX\nRCmzI+ncOX8ncOmlzndcOTmmEc9r0POmbdtMKSynHH2hhYQU3wmUNN+okfnGYtPOQUo6Qgi/09oc\nXa9cabq1XrECfv0VMjLO/d6QEDOYTV4ZqEsX0xV10QZ9+3Zz3iMry/c/T2nq1IFOnczUsaN5bNYs\ncOcupMH3IhtrdpLJGRszgZ25fJUpOxs2bszfAaxcaS5VzevU7hypgLJnatzYjIecN8XFmZPSSUlw\n5Ej+VHA+Pd3pp5ecKTLSNP55O4BOnaB1a//cWCc1fCGEFfJGJGvfHu6/3yxLTzeD2RTcCezaVbbP\nrV+/cKPeqpV5bN68fIPUnGuHkDdt3WrWLSo1FRYvNlOeatXMz11wJ9CmDVStWvZ83iZH+EKIgDl6\ntHApaNUqU4svqVFv0cJ3J33PJTfX7Jx++83stH77zUxOx2CoUsU0+nk7gD59zM9zPqSkI4QQfqK1\n6Y+o6E5g//5zv3fixPxvPuVVngZfbpUoRUJCQqAjFCOZnLExE9iZSzI5U1Impcx5g5tvhhdfNHdT\n79sHhw/D99/Dyy/DgAHmxG5RnTr5PHKJpIYvhBBeVK+eKdn06ZO/LCUF1qzJ/zbQtm1gsklJRwgh\nKiAp6QghhCiVNPilqCh1xECTTM7ZmEsyOWNjpvKQBl8IISoJqeELIUQFJDV8IYQQpfJ5g6+U6quU\n2qKU2qaUetrX2/MWG2t2kskZGzOBnbkkkzM2ZioPnzb4SikX8A7QB2gDDFJKXeTLbXrLmjVrAh2h\nGMnkjI2ZwM5ckskZGzOVh6+P8LsA27XWe7TWZ4DpwM0+3qZXpKSkBDpCMZLJGRszgZ25JJMzNmYq\nD183+I2AvQXm97mXCSGE8DM5aVuKxMTEQEcoRjI5Y2MmsDOXZHLGxkzl4dPLMpVSVwAvaq37uudH\nA1pr/a8i68k1mUIIUUZWdY+slAoCtgI9gYPACmCQ1nqzzzYqhBCiRD7tLVNrnaOUGgXMx5SPPpLG\nXgghAsOKO22FEEL4XkBP2tp2U5ZSKkYptVAptVEptV4p9XCgM+VRSrmUUr8ppWYHOksepVQNpdQM\npdRm979ZVwsyPaaU2qCUWqeU+lQpFRKADB8ppQ4rpdYVWFZTKTVfKbVVKTVPKVXDklyvun9/a5RS\nXyml/DqIYEmZCrz2hFIqVylVy4ZMSqmH3P9W65VSrwQ6k1KqvVJquVJqtVJqhVLq8nN9TsAafEtv\nysoGHtdatwG6AQ9akCnPI8CmQIco4i1gjtb6YqA9ENBynVKqIfAQ0ElrfSmmZHlnAKJMxvxdFzQa\n+FFr3RpYCDzj91Ql55oPtNFadwC24/9cJWVCKRUD9AL2+DkPlJBJKRUP3AS001q3A/4d6EzAq8BY\nrXVHYCzw2rk+JJBH+NbdlKW1PqS1XuN+fhLTgAX8vgH3H39/4MNAZ8njPhK8Wms9GUBrna21Tg1w\nLIAgIFwpFQyEAQf8HUBrvRRILrL4ZmCK+/kU4Ba/hqLkXFrrH7XWue7Z/wExgc7k9gbwlD+z5Ckl\n00jgFa11tnsdh8OX+zRTLpD3TTEKOOdouoFs8K2+KUspFQt0AH4JbBIg/4/fphMuccBRpdRkd6lp\nolKqWiADaa0PAK8Dv2P++FO01j8GMlMB9bTWh8EcWAD1ApynJPcCcwMdQin1B2Cv1np9oLMU0Aq4\nRin1P6XUIiflEz94DPi3Uup3zNH+Ob+dyY1XJVBKRQBfAo+4j/QDmeUG4LD7m4dyTzYIBjoB72qt\nOwGnMGWLgFFKRWGOpJsCDYEIpdRdgcx0FjbtvFFKPQec0Vp/FuAc1YBnMSUKz+IAxSkoGKiptb4C\n+CvwRYDzgPnW8YjWugmm8Z90rjcEssHfDzQpMB+Dg68kvuYuBXwJfKK1/ibQeYArgT8opXYB04Ae\nSqmpAc4E5hvZXq31Kvf8l5gdQCBdD+zSWh/XWucAM4HuAc6U57BSKhpAKVUfOBLgPB5KqWGYkqEN\nO8fmQCywVim1G9Mu/KqUCvQ3or2Yvye01iuBXKVU7cBGYqjW+mt3pi8xZfKzCmSDvxJooZRq6r6S\n4k7AhitQJgGbtNZvBToIgNb6Wa11E611M8y/0UKt9RALch0G9iqlWrkX9STwJ5V/B65QSoUqpZQ7\nU6BOJBf9NjYbGOZ+PhQI1MFEoVxKqb6YcuEftNaZgc6ktd6gta6vtW6mtY7DHFh01Fr7ewdZ9Pf3\nNXAdgPtvvorW+liAM+1XSl3rztQT2HbOT9BaB2wC+mLuxN0OjA5kFneeK4EcYA2wGvgN6BvoXAXy\nXQvMDnSOAnnaY3bcazBHPzUsyDQW08ivw5wcrRKADJ9hThZnYnZCw4GawI/uv/f5QJQlubZjroT5\nzT39v0BnKvL6LqBWoDNhSjqfAOuBVcC1FmTq7s6yGliO2TGe9XPkxishhKgk5KStEEJUEtLgCyFE\nJSENvhBCVBLS4AshRCUhDb4QQlQS0uALIUQlIQ2+EOdBKXWtUurbQOcQwglp8IU4f3Izi6gQpMEX\nlYJS6m6l1C/unj0nuAeUSVNKjXMPmPJDXt8oSqkO7oEl8gYFqeFe3ty93hql1CqlVJz746sXGAjm\nk4D9kEKcgzT44oLnHsTmj0B3bXr2zAXuxvSXv0Jr3RZYQn4PjVOAp7QZFGRDgeWfAm+7l3cHDrqX\ndwAeBi4BmiulbOmwTYhCfDqIuRCW6InpyXOlu1O1UOAwpuHP6+b2P0DeEH81tBlwAkzj/4W7y+xG\nWuvZAFrrLADzcazQWh90z6/B9Pa4zA8/lxBlIg2+qAwUMEVr/VyhhUqNKbKeLrB+WRTsZTIH+X8l\nLCUlHVEZLAAGKKXqgmdA8SaY4RAHuNe5G1iqzTCNx5VSV7qXDwYWazMQzl6l1M3uzwgJ9AhfQpSV\nHImIC57WerNS6nlgvlLKBWQBo4B0oIv7SP8wps4Ppr/6990N+i5MV7RgGv+JSqn/c3/GHSVtznc/\niRDnR7pHFpWWUipNa1090DmE8Bcp6YjKTI52RKUiR/hCCFFJyBG+EEJUEtLgCyFEJSENvhBCVBLS\n4AshRCUhDb4QQlQS0uALIUQl8f8B9XBZG/NbaxkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12014d3d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.plot(train_loss, linewidth=3, label=\"train loss\")\n",
    "pyplot.plot(test_loss, linewidth=3, label=\"valid loss\")\n",
    "pyplot.grid()\n",
    "pyplot.legend()\n",
    "pyplot.xlabel(\"epoch\")\n",
    "pyplot.ylabel(\"loss\")\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mini Batch with Relu and 256 hidden layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Increasing network complexity and introducing non-linearities should improve performance "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "train_loss = []\n",
    "test_loss = []\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    X_train_tf = tf.placeholder(tf.float32,\n",
    "                                    shape=(batch_size, X.shape[1]))\n",
    "    y_train_tf = tf.placeholder(tf.float32, shape=(batch_size, nb_classes))\n",
    "    X_test_tf = tf.constant(np.array(X_test).astype(np.float32))\n",
    "    y_test_tf = tf.constant((np.arange(nb_classes) == y_test[:,None]).astype(np.float32))\n",
    "    \n",
    "    weights1 = tf.Variable(\n",
    "        tf.truncated_normal([X.shape[1], 256]))\n",
    "    biases1 = tf.Variable(tf.zeros([256]))\n",
    "    \n",
    "    weights2 = tf.Variable(\n",
    "        tf.truncated_normal([256, nb_classes]))\n",
    "    biases2 = tf.Variable(tf.zeros([nb_classes]))\n",
    "\n",
    "    h1 = tf.nn.relu(tf.matmul(X_train_tf, weights1) + biases1)\n",
    "    logits = tf.matmul(h1, weights2) + biases2\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, y_train_tf))\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)\n",
    "    \n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    test_prediction_h1 = tf.nn.relu(tf.matmul(X_test_tf, weights1) + biases1)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(test_prediction_h1, weights2) + biases2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Loss at step 0: 9.716219\n",
      "Valid Loss: 8.543947\n",
      "Train Loss at step 1000: 1.845988\n",
      "Valid Loss: 2.275506\n",
      "Train Loss at step 2000: 0.903404\n",
      "Valid Loss: 1.362468\n",
      "Train Loss at step 3000: 0.411469\n",
      "Valid Loss: 1.036987\n",
      "Train Loss at step 4000: 0.260850\n",
      "Valid Loss: 0.879940\n",
      "Train Loss at step 5000: 0.185567\n",
      "Valid Loss: 0.794446\n",
      "Train Loss at step 6000: 0.166375\n",
      "Valid Loss: 0.734401\n",
      "Train Loss at step 7000: 0.136283\n",
      "Valid Loss: 0.694401\n",
      "Train Loss at step 8000: 0.100913\n",
      "Valid Loss: 0.662252\n",
      "Train Loss at step 9000: 0.101982\n",
      "Valid Loss: 0.641285\n",
      "Train Loss at step 10000: 0.071903\n",
      "Valid Loss: 0.620439\n",
      "Train Loss at step 11000: 0.074838\n",
      "Valid Loss: 0.603872\n",
      "Train Loss at step 12000: 0.061306\n",
      "Valid Loss: 0.589687\n",
      "Train Loss at step 13000: 0.064904\n",
      "Valid Loss: 0.579660\n",
      "Train Loss at step 14000: 0.047869\n",
      "Valid Loss: 0.569570\n",
      "Train Loss at step 15000: 0.071436\n",
      "Valid Loss: 0.559656\n",
      "Train Loss at step 16000: 0.040468\n",
      "Valid Loss: 0.553105\n",
      "Train Loss at step 17000: 0.067016\n",
      "Valid Loss: 0.547071\n",
      "Train Loss at step 18000: 0.032301\n",
      "Valid Loss: 0.541087\n",
      "Train Loss at step 19000: 0.036903\n",
      "Valid Loss: 0.535128\n"
     ]
    }
   ],
   "source": [
    "num_steps = 20000\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    for step in range(num_steps):\n",
    "        # Pick an offset within the training data, which has been randomized.\n",
    "        # Note: we could use better randomization across epochs.\n",
    "        offset = (step * batch_size) % (len(y_train) - batch_size)\n",
    "        # Generate a minibatch.\n",
    "        batch_data = np.array(X_train)[offset:(offset + batch_size), :]\n",
    "        batch_labels = y_train[offset:(offset + batch_size)]\n",
    "        batch_labels = (np.arange(nb_classes) == batch_labels[:,None]).astype(np.float32)\n",
    "        # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "        # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "        # and the value is the numpy array to feed to it.\n",
    "        feed_dict = {X_train_tf : batch_data, y_train_tf : batch_labels}\n",
    "        _, l, predictions = session.run(\n",
    "              [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "        if (step % 1000 == 0):\n",
    "            train_loss.append(l)\n",
    "            test_loss.append(log_loss(y_test, test_prediction.eval()))\n",
    "            print('Train Loss at step %d: %f' % (step, l))\n",
    "            print('Valid Loss: %f'% log_loss(y_test, test_prediction.eval()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEPCAYAAABGP2P1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VNX5+PHPM9kXSCBAgGASNkEWDahQUCEuKGit2p9Q\nUbFgi0ulrrXaBZfWfmu1pRbbolbZ3Ev9un61AmoEpCpWURBQWQLIEtaQjezn98edzJJMYJLcyZ2Z\nPO/X677mnnvv3HlyCPPknHPvuWKMQSmllGrgcjoApZRS4UUTg1JKKT+aGJRSSvnRxKCUUsqPJgal\nlFJ+NDEopZTyE9LEICIJIvKRiHwmIutE5N5mjpsrIt+IyFoRyQtlTEoppY4tNpQnN8ZUicjZxpgK\nEYkBPhCRt4wxHzccIyKTgP7GmIEiMhp4DPhOKONSSinVvJB3JRljKtyrCViJqPEddZcAi93HfgSk\niUhmqONSSikVWMgTg4i4ROQzYC+wzBizptEhWcBOn/Iu9zallFIOaI8WQ70xZgTQBxgtIkNC/ZlK\nKaVaL6RjDL6MMSUi8h4wEdjgs2sXcIJPuY97mx8R0UmdlFKqFYwx0pLjQ31VUjcRSXOvJwETgE2N\nDnsNuMZ9zHeAYmNMUaDzGWNatMSe8Re4D7gPbnh1VovfH83Lvffe63gM0bRofWpdhuvSGqFuMfQC\nFomICysJvWiMeVNErgeMMeYJd/lCEdkMlAMz7PrwOJKoda+XVR2167RRobCw0OkQoorWp320Lp0X\n6stV1wEjA2x/vFF5Vig+P06SaEgH5dWaGJRSKhhRfedzvCR51is0MfiZPn260yFEFa1P+2hdOi+6\nE4Mr2bNeUaOJwVd+fr7TIUQVrU/7aF06L6oTQ4LLt8VQcYwjO56CggKnQ4gqHbU+c3NzERFdwmDJ\nzc217d+13S5XdUJCjDcxHK3VFoNSdtu+fXurr3xR9hJp0RWpxxTVLYZETQzN0ua6vbQ+VTSJ6sSQ\nFOtNDFV1mhiUUioYUZ4YvIPPlZoY/HTUPvFQ0fpU0SS6E0Ocb4tBB5+VUi1z44038rvf/a5V7z37\n7LOZP3++zRG1j6gefE6O9yaGaqMtBl/aJ24vrc/w07dvX5566inOOeecVp9j3rx5NkYUOaK6xZAc\nn+hZrzFV1Jt6B6NRSoWTuro6p0MIW1GdGBITBGq8yaGyttLBaMKL9onbS+szvFxzzTXs2LGDiy++\nmM6dO/PHP/6R7du343K5mD9/Pjk5OZx77rkATJkyhV69etGlSxfy8/PZsME7+fOMGTO45557AHj/\n/fc54YQTmDNnDpmZmWRlZbFw4cKg4jHG8MADD5Cbm0vPnj2ZPn06JSUlAFRVVTFt2jS6detGly5d\nGD16NPv37wdg4cKF9O/fn86dO9O/f3+ef/55G2upedGdGBKBWp9LVvXuZ6XajYi9S0ssXryY7Oxs\n3njjDUpKSvjZz37m2bdixQo2bdrE22+/DcCFF17Ili1b2LdvHyNHjuSqq65q9rx79+6ltLSU3bt3\n8+STT3LTTTdx5MiR48azYMECFi9ezPvvv8/WrVspLS3lpz/9KQCLFi2ipKSEXbt2cejQIR577DGS\nkpKoqKjglltu4e2336akpITVq1eTl5fXsopopahODAkJQI3vtBg6AN1A+8TtpfUZnhrffCci3H//\n/SQlJZGQkABYczMlJycTFxfHPffcw+eff05paWnA88XHxzN79mxiYmKYNGkSqampfPXVV8eN47nn\nnuP2228nJyeH5ORkfv/73/PCCy9QX19PXFwcBw8e5Ouvv0ZEGDFiBKmpqQDExMSwbt06KisryczM\n5KSTTmpjjQSnAyQGvclNKeXVp08fz3p9fT133303AwYMID09nb59+yIiHDhwIOB7MzIycLm8X5vJ\nycmUlZUd9zN3795NTk6Op5yTk0NNTQ1FRUVMmzaNCy64gCuuuII+ffpw9913U1dXR3JyMi+++CLz\n5s2jV69eXHzxxUElITtEdWLQrqTmaZ+4vbQ+mzLG3qWlmpsiwnf7c889x+uvv867775LcXExhYWF\nbXrATXN69+7N9u3bPeXt27cTFxdHZmYmsbGxzJ49my+//JLVq1fz+uuvs3jxYgAmTJjA0qVL2bt3\nL4MGDWLmzJm2xtWcqE4M2mJQquPq2bMnW7du9dvW+Au/tLSUhIQEunTpQnl5Ob/4xS9snXOowdSp\nU/nzn/9MYWEhZWVl/OpXv+KKK67A5XJRUFDA+vXrqa+vJzU1lbi4OFwuF/v27eO1116joqKCuLg4\nUlNTiYmJsT22QKI/MWiLISDtE7eX1mf4ufvuu/ntb39L165dmTNnDtC0FXHNNdeQnZ1NVlYWw4YN\nY+zYsS36jGMlEd991157LdOmTWPcuHH079+f5ORk5s6dC1gD2pdffjlpaWkMHTqUs88+m2nTplFf\nX8+cOXPIysqiW7durFixot3uq5BImRlRRExLY33ySZhZcCEMfAuAN6a+wUUnXhSK8JTqkEREZ1cN\nE839W7i3t6gZFP0tBu1KCkj7xO2l9amiSfQnBu1KUkqpFonqxJCYiLYYmqF94vbS+lTRJKoTg7YY\nlFKq5aI/MeidzwFpn7i9tD5VNInqxKBdSUop1XJRnRi0K6l52iduL61PFU2iPzFoi0EppVok+hOD\nthgC0j5xe2l9Ro+G5y40GDZsGCtWrAjq2MZcLleTaTkiQVQ/2rPxGENFrQ4+K6WOz3c6i/Xr1wd9\nbEv2hbOQthhEpI+IvCsiX4rIOhG5OcAx40WkWEQ+dS+/tuvzG1+VpC0GL+0Tt5fWpwokUqcLCXVX\nUi1wuzFmKDAGuElEBgc4boUxZqR7ecCuD2/SlaRjDEp1CA899BCTJ0/223bLLbdw6623AtYjM4cM\nGULnzp0ZMGAATzzxRLPn6tu3L++++y4AlZWVTJ8+na5duzJs2DDWrFkTdEwlJSVcc8019OjRg759\n+/K73/3Os2/Lli3k5+eTnp5Ojx49mDp1qmffbbfdRmZmJmlpaZxyyil+jx4NlZB2JRlj9gJ73etl\nIrIRyAI2NTo0JO2tJperaovBo6CgQP/KtZHWZ1Nyv73/rc29wf/1fcUVV/Cb3/yG8vJyUlJSqK+v\nZ8mSJbz66qsAZGZm8uabb5Kbm8vKlSuZOHEio0aNOu6jM++77z62bdvGtm3bKCsrY+LEiUHHNGvW\nLEpLSyksLGT//v2cf/759O7dmxkzZjB79mwuuOACCgoKqK6u5pNPPgFg6dKlrFq1is2bN9OpUye+\n+uor0tPTg/7M1mq3wWcRyQXygI8C7B4jImtF5P9EZIhdnxkbiw4+K9UBZWdnM3LkSF5++WUA3nnn\nHVJSUjj99NMBmDRpErm5uQCcddZZnH/++axcufK4512yZAm//vWvSUtLIysri5tvbtI7HlB9fT0v\nvvgiDz74IMnJyeTk5HDHHXfw9NNPAxAXF8f27dvZtWsX8fHxnum/4+LiKC0tZcOGDRhjGDRoEJmZ\nmS2tjhZrl8QgIqnAv4BbjDGNn4P3XyDbGJMH/BV4xb7PhXiXNzGUV+vgcwP969ZeWp/hZ+rUqTz/\n/PMAPP/881x55ZWefW+99RZjxowhIyODLl268NZbbzX7OE9fu3fv9ns0qO/jOo/lwIED1NbWkp2d\n7ffeXbt2AVbXV319PaNGjWL48OEsWLAAgLPPPptZs2Zx0003kZmZyQ033BDUo0TbKuRXJYlILFZS\neNoY82rj/b6Jwhjzloj8XUS6GmMONT52+vTpniyfnp5OXl6e5z9kw+WCjcsJrhOoBtgGh/Z5T9nc\n8VrWspaDLx9LS7p+QmHy5Mn87Gc/Y9euXbz88st8+OGHAFRXV3P55ZfzzDPPcMkll+ByubjsssuC\nGiju1asXO3fu5KSTTgLwe1znsXTr1s3TKhg8eLDnvVlZWYDVtdUwzvHBBx9w3nnnMX78ePr168es\nWbOYNWsWBw4cYPLkyTz88MPcf//9zX5WQUEBCxcuBPB8X7ZYw/NNQ7UAi4E5x9if6bM+Cihs5jjT\nGt367jLch+E+TI8/9GzVOaLRe++953QIUaWj1mdr/1+2l0mTJpkJEyaYkSNHeraVlpaa2NhYs2LF\nCmOMMW+++aZJTk42s2fPNsYYU1BQYE444QTP8bm5ueadd94xxhhz1113mfz8fHP48GGzc+dOc/LJ\nJ/sd25iImC1bthhjjJk2bZr5/ve/b0pLS01hYaEZPHiwmT9/vjHGmCVLlphvv/3WGGPM+vXrTXJy\nstm2bZtZs2aN+eijj0xNTY0pKyszEydONPfdd1/Az2ru38K9vUXf26G+XPUM4CrgHBH5zH056kQR\nuV5ErnMfdrmIrBeRz4BHgB/YGUNijF6VpFRHdeWVV/LOO+9w1VVXebalpqYyd+5cJk+eTNeuXXnh\nhRe45JJLmj2H770I9957L9nZ2fTt25eJEydyzTXXHPPzfd87d+5ckpOT6devH+PGjePqq69mxowZ\nAKxZs4bRo0fTuXNnLr30UubOnUtubi4lJSXMnDmTrl270rdvX7p168add97Z2uoIWlQ/2hNg4EmV\nbL7CSg5xrniqZ1fZHZpSHZY+2jN86KM9WyAxLgHcdVJTX01dfZ3DESmlVHiL/sSQIDqRXgA6t4+9\ntD5VNIn+xJCI3suglFItEPWJQafeDkyvu7eX1qeKJh0jMWiLQSmlghb1iUEf7xmY9onbS+tTRZOo\nfh4DNG0xVNTotBhK2SUnJydinzkQbYKdniMYHSMx6DMZmtA+cXt11PosLCx0OgQVAtqVpJRSyk/U\nJwYdfA5M+8TtpfVpH61L53WMxKAtBqWUClrUJ4bGN7jp4LOlo/aJh4rWp320Lp0X9YkhIQGo6uQp\nF5UVOReMUkpFgI6RGPZ6n+O6aucq54IJI9qPay+tT/toXTov6hNDYiKwfbynvHrnamrqapwLSCml\nwlzUJ4aEBOBINhzuC1hjDJ/s/sTZoMKA9uPaS+vTPlqXzusYiQGg0NtqeH/7+84Eo5RSEaDjJIbt\nmhh8aT+uvbQ+7aN16byoTwyJie4VnxbDqh2rqK2vdSYgpZQKc1GfGDwthuJcEqtOAKCsuoxP93zq\nXFBhQPtx7aX1aR+tS+d1nMSA0PmwT3dSoXYnKaVUIFGfGDxdSUDKfh1naKD9uPbS+rSP1qXzoj4x\neFsMkLDHmxhW7lhJXX2dAxEppVR4E2OM0zEERURMa2LdsAGGDrXWBw02lPw4iz1lewD473X/ZWSv\nkXaGqZRSYUVEMMa06GlKUd9i8O1Kqq4SxufqOINSSh1L1CcG366kykoYn6PjDKD9uHbT+rSP1qXz\nOlRiqKryTwwrd6yk3tQ7EJVSSoWvqB9jKCuDTu5Zt1NSoLTU0PNPPdlXvg+Az2/4nJMzT7YzVKWU\nChs6xhBA4xaDiPh3J+k4g1JK+QlpYhCRPiLyroh8KSLrROTmZo6bKyLfiMhaEckLdExrxcaCuHNl\nbS3U1ek4A2g/rt20Pu2jdem8ULcYaoHbjTFDgTHATSIy2PcAEZkE9DfGDASuBx6zMwAR/yuTqqrw\nvzJp+/tESneaUkq1h3YdYxCRV4BHjTHv+Gx7DHjPGPOiu7wRyDfGFDV6b6vGGAC6dIHiYmv90CFI\nS6+nx8M9OHj0IADrb1zP0B5DW3VupZQKZ2E9xiAiuUAe8FGjXVnATp/yLvc22zS+ZNUlLsbljPNs\n66jdSUopFUhse3yIiKQC/wJuMcaUtfY806dPJzc3F4D09HTy8vI8MzE29EsGKltdSVa5qsra3/tg\nb9gG9LUSw5DyIc2+PxrLjzzySND1p2Wtz/Ys+44xhEM8kVYuKChg4cKFAJ7vy5YKeVeSiMQCbwBv\nGWP+EmB/466kTcB4O7uSBg2Cr7+21jdtsspr965lxOMjAMhMyWTPHXsQaVFrK6IVFBR4fqlU22l9\n2kfr0l7h2pU0H9gQKCm4vQZcAyAi3wGKGyeFtmrclQQwvMdw0hPTASgqL+Lrg1/b+ZFhT//j2Uvr\n0z5al84L9eWqZwBXAeeIyGci8qmITBSR60XkOgBjzJvANhHZDDwO/MTuOBpflQQQ44rhrOyzPNt1\nnEEppSwhTQzGmA+MMTHGmDxjzAhjzEhjzL+NMY8bY57wOW6WMWaAMeYUY4ztj1ZrfJNbg458P4Nv\nP65qO61P+2hdOi/q73yGwF1JQJOZVvV+BqWU6iCJIVBXEkBezzw6xVsTKe0q3cXWw1vbOTLnaD+u\nvbQ+7aN16bwOkRia60qKdcVyVo6OMyillK8Olxh8u5Kg444zaD+uvbQ+7aN16bwOkRia60oC/8RQ\nUFjQPgEppVQY6xCJobmuJICRvUaSEpcCwI4jOygsLmy/wByk/bj20vq0j9al8zpEYkhJ8a4XNbp1\nLi4mjjOyz/CU9fkMSqmOrkMkhjyfJzx88EHT/R1xnEH7ce2l9WkfrUvndYjEcJb3wiP+8x/rgT2+\nOmJiUEqp5kT9M58b5OTAjh3W+scfw+mne/dV11WT/mA6R2uPArDztp306dynLeEqpVRYCNdJ9MKC\nb6th5Ur/ffEx8Yw5YYynrOMMSqmOTBODW0frTtJ+XHtpfdpH69J5HTIxrFoFjXulOlpiUEqp5nSY\nMQZjoHt3OGg95pkNG+Ckk7z7K2srSX8wnao660aH3bfvplenXm0JWSmlHKdjDMcgAmee6S037k5K\njE1kdJ/RnvKK7SvaKTKllAovHSYxgH930ooA3/sdaXoM7ce1l9anfbQunddhE0OgAej83HzPuo4z\nKKU6qg4zxgBQUwNdukB5uVXevh2ys737K2oqSH8wnZr6GgCKflZEj5QebfpMpZRyko4xHEdcHIzx\n3q7QpNWQHJfMqKxRnrKOMyilOqIOlRighfczRPGNbtqPay+tT/toXTpPE0Mjfs+B1nEGpVQH1KHG\nGAAqKiA93RpvADhwADIyvPvLqstIfzCdOlNn7b/zABnJGQHOpJRS4U/HGIKQnAynnuotr1rlvz81\nPpXTep/mKa/cEaBZoZRSUazDJQbQcQbQfly7aX3aR+vSeR0+MQS80U3HGZRSHVhQYwwicguwACgF\nngRGAHcbY5aGNjy/GGwZYwA4dMg7rhATA8XFkJrq3V9SVUKXP3Sh3tQjCIfuOkR6Yrotn62UUu0p\nlGMM1xpjSoDzgS7ANODBFsYXNrp2heHDrfW6OvjwQ//9nRM6M6LnCAAMhlU7Gg1EKKVUFAs2MTRk\nmwuBp40xX/psi0gtGWeIxnmTtB/XXlqf9tG6dF6wieG/IrIUKzG8LSKdgPrjvUlEnhKRIhH5opn9\n40WkWEQ+dS+/Dj70ttH7GZRSKrBgxxhcQB6w1RhTLCJdgT7GmIBf+D7vOxMoAxYbY04OsH88cIcx\n5ntBxGDbGAPArl3Qx/1Y56Qka5whPt67//DRw2Q8lIHB4BIXh+86TOeEzrZ9vlJKtYdQjjGMAb5y\nJ4WrgV8DR473JmPMKuDwcQ5zpEsqKwv69rXWjx6FTz/1398lqQun9DwFgHpTzwc7PmjnCJVSyhnB\nJoZ5QIWInALcAWwBFtsUwxgRWSsi/yciQ2w6Z1A68nOgtR/XXlqf9tG6dF5skMfVGmOMiFwC/NUY\n85SI/MiGz/8vkG2MqRCRScArwInNHTx9+nRyc3MBSE9PJy8vj/z8fMD7y9SSco8eAFb55ZcLOP10\n//0ZRd6pMF57+zUmxk5s0+eFU3nt2rVhFU+kl7U+tRwu5YKCAhYuXAjg+b5sqWDHGN4H/g1cC5wF\n7AM+N8YMD+K9OcDrgcYYAhy7DTjVGHMowD5bxxgAvvoKBg+21rt0seZNcvm0oQ5UHKD7w90BiHXF\nUnxXMSnxKbbGoJRSoRTKMYYfAFVY9zPsBfoADwcbF82MI4hIps/6KKxE1SQphMqJJ+JuNcDhw/Dl\nl/77uyV3Y1iPYQDU1teyeufq9gpNKaUcE1RicCeDZ4E0EfkuUGmMOe4Yg4g8B6wGThSRHSIyQ0Su\nF5Hr3IdcLiLrReQz4BGsBNRuRDruOEND01PZQ+vTPlqXzgsqMYjIFOBjYDIwBfhIRC4/3vuMMVca\nY3obYxKMMdnGmAXGmMeNMU+49//NGDPMGDPCGDPWGPNRW36Y1uioiUEppZoT7BjD58AEY8w+d7k7\nsNwYc0qI4/ONwfYxBrAuU22YhjsrC3butFoSDYrKiuj5p54AxMfEU3xXMUlxSbbHoZRSoRDKMQZX\nQ1JwO9iC94a1U06BTp2s9V27oLDQf39maiaDu1kj1NV11Xz4baOJlZRSKsoE++X+bxF5W0Smi8h0\n4P+AN0MXVvuJiYGxY73ljjJvkvbj2kvr0z5al84LdvD5TuAJ4GT38oQx5q5QBtaedJxBKaW8Otwz\nnwNZsQLGu7/7TzzRur/B1+7S3WTNyQIgISaBQ3cdIjkuOSSxKKWUnWwfYxCRUhEpCbCUikhJ28IN\nH6NGeSfQ+/prKCry39+7U29OzLBuyK6qq2L2u7PbOUKllGo/x0wMxphOxpjOAZZOxpiomWo0MdFK\nDg1WBXguzx1j7vCs//nDP0f8WIP249pL69M+WpfOi4ori+xwvHGGmSNnMnHARMB6qtsPX/khRyqP\nO8GsUkpFHB1jcHvrLbjwQmt95Ej473+bHrO7dDfD5w3n0FFr1o4fnvJDFl66MGQxKaVUW4XyPoao\nN3as98a2tWuhJMAISu9OvZl30TxPedHni3h548vtFKFSSrUPTQxuaWnWzW4A9fXwn/8EPm7K0Clc\nOfxKT/m6N65jb9nedojQXtqPay+tT/toXTpPE4OP440zNPjrpL+S1cm6fPVAxQFmvj6TSOmSU0qp\n49ExBh9LlsCUKdb6uHHw/jHuZVu+dTkTnp7gKf/j4n/w45E/Dml8SinVUq0ZY9DE4GPvXujVy1pP\nSIAjR6zX5tz81s08+vGjAKTGp/L5DZ/Tr0u/kMaolFItoYPPbdSzJwwcaK1XVcGaNcc+/sHzHmRQ\nxiAAyqrLuObla6irrwtxlPbQflx7aX3aR+vSeZoYGgl2nAEgOS6Zpy97mhiJAeCDnR/wx9V/DGF0\nSikVetqV1MjChTBjhrU+aRK8GcQcsvcX3M99798HQJwrjk+u+4STM4/7iGullAo5HWOwwZYtMGCA\ntd65Mxw6ZE3NfSw1dTWcMf8M1uy2+p6G9xjOmplrSIg9xgCFUkq1Ax1jsEG/ft4B6JISWLfu+O+J\ni4nj6cueJinWerLbun3ruOe9e0IYZdtpP669tD7to3XpPE0MjYi0bJyhwaBug3howkOe8sOrH2bl\n9iDfrJRSYUS7kgL461/hpz+11idPhn/+M7j31Zt6Jj4zkWVblwHQN70vn9/wOZ0SOoUoUqWUOjbt\nSrJJ4xZDsPnIJS4WXLKA9MR0ALYVb+O2t28LQYRKKRU6mhgCGDYM0q3vdvbuhc2bg39vVucs/n7h\n3z3lpz57ite+es3mCNtO+3HtpfVpH61L52liCCAmBs44w1sOdpyhwdThU/nB0B94yjNfn8n+8v02\nRaeUUqGliaEZrRmA9vX3i/5O7069AdhXvo/r3rgurCbay8/PdzqEqKL1aR+tS+dpYmhGWxND16Su\nzP/efE/5lU2vsOjzRTZEppRSoaWJoRmnnWY9Cxqsm9727Gn5OS4YcAE/Oe0nnvLNb91MYXGhPQG2\nkfbj2kvr0z5al87TxNCM+HgYPdpbbk2rAeChCQ8xsKs1M19pdSnTX5lOvam3IUKllAqNkCYGEXlK\nRIpE5ItjHDNXRL4RkbUikhfKeFqqrd1JACnxKX4T7b2//X3ufe9ex5OD9uPaS+vTPlqXzgt1i2EB\ncEFzO0VkEtDfGDMQuB54LMTxtIgdiQFgdJ/R/PKsX3rKD6x8gInPTGRXya42RKeUUqER0sRgjFkF\nHD7GIZcAi93HfgSkiUhmKGNqiTFjwOWuoS++gOLi1p9r9rjZjMsZ5ykv27qM4fOG888vg7yt2mba\nj2svrU/7aF06z+kxhixgp095l3tbWOjUCUaOtNaNgQ8+aP254mLiWHr1Un4+9ucI1t3physP84N/\n/YBpL0/jSOURGyJWSqm2i3U6gJaYPn06ubm5AKSnp5OXl+fpj2z4K8Pu8lln5fPJJwAFPPssXHRR\n2873hwl/4KITL2LKw1MoKiuCvvDMF8+w9J2l/OLMX3DrFbeG9Ofx7b8tKCgIef11lHLDtnCJJ5LL\n+fn5YRVPpJULCgpYuHAhgOf7sqVCPomeiOQArxtjmjy5RkQeA94zxrzoLm8CxhtjigIc226T6Pl6\n+WX4/vet9bFj29Zq8HWk8gg3//tmFn++2LNNEO4YcwcPnPOAPstBKWWLcJ1ET9xLIK8B1wCIyHeA\n4kBJwUlnnuldX7MGjh6157xpiWksunQRSyYvoWtSVwAMhj/+54+c/o/TWVcUxIMg2qDhLwxlD61P\n+2hdOi/Ul6s+B6wGThSRHSIyQ0SuF5HrAIwxbwLbRGQz8Djwk2OczhHdu8PgwdZ6TQ18/LG95798\nyOWsu3EdF/T3Xry1bt86TvvHafxp9Z8cv6xVKdXx6PMYgnDddfCPf1jrv/0t/PrX9n+GMYa/rfkb\ndy67k8raSs/2/Nx8Fl26iOy0bPs/VCkV9cK1Kyni2XU/w7GICLNGzeKz6z/j1F6nerYXFBZw8ryT\nefaLZ8NqEj6lVPTSxBAE38SwejXU1obuswZ3G8zqH63mV2f9CpdY/zxHqo5w9ctXM/WlqRw6esiW\nz9F+XHtpfdpH69J5mhiCkJMDJ5xgrZeVwdtvh/bz4mPieeCcB1g5YyX9uvTzbH/xyxc5ed7JLNuy\nLLQBKKU6NB1jCNIdd8CcOdZ6fj689177fG5pVSm3vX0bT332lN/2M044gxl5M5gydIo+U1op1azW\njDFoYgjSjh3Qrx/U1VnlNWusqbnbyyubXmHm6zM5UHHAb3tyXDKTh0xmRt4MxuWMQ6RF//5KqSin\ng88hlJ0NV1zhLT/8cPt+/qWDL2X9jeu5YtgVxLq8N6xX1FSw6PNF5C/KZ+CjA3lgxQPsPLLzGGey\naD+uvbSSRn9iAAAVR0lEQVQ+7aN16TxNDC1w553e9X/9C7Zubd/Pz0zN5Pn/9zy7bt/Fn87/E0O7\nD/Xbv+XwFma/N5ucR3K44JkLeGH9C36XviqlVDC0K6mFzj8flrnHfmfNgkcfdS4WYwyf7P6EBWsX\n8Pz65ymubDr9a3piOlOHTeXaEddyaq9TtatJqQ5GxxjawbJlVnIASEqyxh66dXM2JoDK2kpe2fQK\n8z+bz/KtyzE0rathPYYxI28GV598NT1SejgQpVKqvWliaAfGWFNxr11rle+/H+65x9mYGttxZAeL\nP1/MgrUL2Hq4aX9XrCuWU6tOZep3pzKh/wRO6naStiTayHdmVdU2Wpf20sTQTp59Fq6+2lrv3h22\nb7daD+Gm3tSzcvtKFqxdwJINS6ioqfDu3Ab0tVZ7pfbivH7nMaHfBM7tdy69O/V2JN5Ipl9m9tG6\ntJcmhnZSUwP9+8NO98U/8+bBDTc4G9PxlFaVsmTDEuZ/Np8Pdh577vAh3YdwXt/zmNB/AuNzxut9\nEkpFME0M7eiRR+C226z1AQNg0yaIiXE2pmBtO7yNpVuWsmzrMt7d9i6HK5t/+mqsK5bRWaOZ0G8C\n5/U7j1FZo4iLiWvHaJVSbaGJoR2VlVnTZDQ8B/qll7wP9IkEDc31uvo6Ptv7Gcu2LGP5tuWs2rGK\n6rrqZt+XGp9Kfm4+E/pN4Dt9vsPQ7kNJiU9px8jDk3Z/2Efr0l6tSQwR9WjPcJKaCjfeCL//vVV+\n+GG47DKItDHcGFcMp/U+jdN6n8YvzvoFFTUVrNqxiuVbl7N863I+2/uZ3/Fl1WW88fUbvPH1G4D1\n1Ln+XfszvMdwTs482fPar0s/YlwR0oRSSvnRFkMb7NkDublQ7f4De+VK/ye+RYP95ft5d9u7LN+6\nnGVbl7H9yPag3pcUm8SwHsO8CSPTeu2WHAbX9irVgWhXkgN+/GN4yj2/3fe+B6++6mw8oWSMYcvh\nLSzfupyCwgK+KPqCrw9+TZ2pC/ocPVN7eloWw3sMZ1C3QQzoOoCMpAy9ZFapENDE4ICNG2HIEP9y\nw6NAw5ld/biVtZVs3L+RdfvW8UXRF57XvWV7W3Se9MR0BnQdwMCuAz2vAzMGRkzS0H5x+2hd2kvH\nGBxw0klw8cXw+utW+U9/8j4GtCNIjE1kRK8RjOg1wm/7/vL9rNu3jnVFVqL4Yt8XfLnvS47WHg14\nnuLKYj7Z/Qmf7P6kyb5ITxpKRRptMdhg5UoYN85aj4+3bnjr2dPZmMJRXX0dWw5v8SSLDQc28M3B\nb9h8aDPlNeWtOmen+E70TO3pt2SmZDbZ1iOlh15mqzok7UpyiDEwZgx89JFV/uUv4Xe/czamSGKM\nYW/ZXjYf2sw3h76xksXhzW1OGo1lJGU0SRiBEklGcobnsapKRTpNDA566SW4/HJrPT3dmlyvUxjf\nMBwp/bgNSeObQ1aSCFXS8BUjMfRI6WEljVR30kjp6V92L2kJaYhIxNRnJNC6tJeOMTjo0kutO6A3\nb7ZuenvqKbj1VqejinwiQq9OvejVqRfjcsb57TPGcLjyMHvL9lJUVsTesr3epdy7XlRWxL7yfQFn\nnA2kztSxp2wPe8r2HPfY+Jh4eqb2JGFnArk7c+ma1JWMpAwykjP8XrsmdfWspyWmaYtEhTVtMdho\n3jz4yU+s9exsK0nEabd2WKitr+VAxQG/5FFUVsSesj0UlRf5bTvWFCF2cInLL4E0rHdJ7EJ6YnrA\npUuStS81PlWTimoR7Upy2NGjVkI44H4s87PPwpVXOhuTarmq2iqKyov8WiF+ycNnvay6rF1jc4mL\ntIS0pkkjIZ3OCZ1JjU8lJT6FlLgUz3pqfCopcSl+6w374mPi2zV+1f40MYSB+++H++6z1vPy4NNP\nw3OaDO3HtUd5dTlF5UUsXb6U3BG5HKw4yMGjBz2vh44e8isfrDhIaXWp02F7xLnimk0knnJc4O3H\nek9SbFKrLyPW30176RhDGLjpJvjDH6zWw9q18M47cN55TkelQiUlPoV+8f0Y3H0w+QPyg3pPdV01\nh48ebpJAiiuL/ZbDlYebbLO7hVJTX+M5t50E8SSPxq0WTwJpZl/htkJKepWQEJNAYmwiCbEJfuuJ\nsYkkxCR41mNd+jVmN20xhMCsWfC3v1nr558Pb7/tbDwqetTU1XCk6kjTJHL0MCVVJZTXlFNeXU5Z\ndRnlNd7XJtvc5ZZMZxKuXOJqkiwa1hte42Pim2xLiAlifzP7jrctnJJVWHYlichE4BHABTxljPlD\no/3jgVeBhmdQ/q8x5oEA54mYxLB1KwwcCPX1VnntWjjlFGdjUqoxYwzVddV+yaJx4mi2fJz3VNZW\nOv3jOcolrmaTSHxMfJME5Let0fty03P50cgftTqWsEsMIuICvgbOBXYDa4ArjDGbfI4ZD9xhjPne\ncc4VMYkBYMoUWLLEWr/6anj6aWfjaUz7ce2l9emvrr7O01JpnER8E0jj9fKacrat3UbaoDQqayup\nqquyXmur/NZ999Wbeqd/3JAa02cMq3+0utXvD8cxhlHAN8aY7QAi8gJwCbCp0XFhODzbNnfe6U0M\nL7wA//M/1oN9lOoIYlwxdE7oTOeEzi1+b0F6y5JsbX1tk4TRkEgCvVbXVR/zmOq6au+25rYfY1tV\nbVXQ98wEIyE2wbZzBSvULYb/B1xgjLnOXb4aGGWMudnnmPHAS8C3wC7gTmPMhgDniqgWA0B+Prz/\nvrV+++3WBHtKqehXW1973GQU7LbstGxmjJjR6ljCscUQjP8C2caYChGZBLwCnOhwTLa4805vYnji\nCZg925ouQykV3WJdscTGx5JCZD72NtSJYReQ7VPu497mYYwp81l/S0T+LiJdjTGHGp9s+vTp5Obm\nApCenk5eXp6nyVlQUAAQVuWkJBgyJJ8NG6CsrICf/xyeeCI84nvkkUfCvv4iqaz1aV+5YT1c4om0\nckFBAQsXLgTwfF+2VKi7kmKAr7AGn/cAHwNTjTEbfY7JNMYUuddHAf80xuQGOFfEdSUBLFgA115r\nrffsCYWFkND+XYZNFOhgqa20Pu2jdWmvsLsqCTyXq/4F7+WqD4rI9YAxxjwhIjcBNwI1wFHgNmPM\nRwHOE5GJoaoK+va1ng8N1uR6DYlCKaVCLSwTg10iNTGAdSf03Xdb6yedBOvXg0vnQVNKtYPWJAb9\nemoH118PqanW+saN8OabzsYD3j5JZQ+tT/toXTpPE0M7SE+H667zlmfNsh4HqpRS4Ui7ktrJzp3W\ng3yqq62yCNxyi/UI0ORkZ2NTSkUv7UoKYyecAM8/D53dN4IaA488Yk3N/cEHzsamlFK+NDG0o+9/\nH778EiZO9G775hs46yy44w5rqu72ov249tL6tI/WpfM0MbSzPn2swecnn/RvPcyZY7Ue/vMfZ+NT\nSikdY3DQzp3w4x/D0qXebS6XNa/Sb34DSUnOxaaUig56H0MEMsZqPdxxB5T6PPFx8GBYuBBGj3Ys\nNKVUFNDB5wgkAjNnwrp1/o8A3bQJxo6Fu+6CyhA880T7ce2l9WkfrUvnaWIIEzk5VpfSY495b4ar\nr4eHHoKRI2HNGmfjU0p1HNqVFIYKC+FHP4J33/Vuc7ng5z+H++4Lj0n4lFKRQccYokh9PTz+uPVM\nh/Jy7/ahQ62xh9NOcyw0pVQE0TGGKOJywY03WmMPvjMQf/kljBoF55xjdTvt29e682s/rr20Pu2j\ndek8TQxhrm9feOcdePRR79QZxsB771mJo1cvOPdcq3Wxf7+zsSqlooN2JUWQLVvgpz+Ff//bSg6N\nuVxw9tkwZQpcdhl0797+MSqlwouOMXQQu3fD//4v/POfsGpV4CQRE+OfJLp1a/84lVLO0zGGDqJ3\nb2vq7hUr4NtvYe5cOPNM/2Pq6mD5cmu675494fzzrRvpDh609ms/rr20Pu2jdek8bTFEkV274KWX\nrJZEczO2xsRYYxI5OQWMHZtP9+7Qo4d30Wk4WkefU2wfrUt7aVeS8vj2W2+SWL06+PelptIkWTQs\njbdnZuojSpUKd5oYVEA7d3qThJ2zt8bHW1dN9evnv/Tvb21vuINbKeUcTQzquHbuhNdegxUrCkhI\nyGffPvyWmhr7PqtHD2+iaJw4evWKrtaGdn/YR+vSXq1JDLGhCkaFpxNOgJtusu6gbvx/zxg4csS6\nH6JxwmhYGvbt3esdyG5Ow3s+/LDpvoQEq1UxeLB3OekkGDQI0tJs+3GVUq2gLQbVaiUlsG0bbN1q\nLVu2eNcLC1vf+ujVy5sofJNGVpY1G61SKnjalaTCRl2ddZWUb7LwTSDHa20EkpLinygGD7aeiAfW\n3FLGWIvvejDljAyri6tbN008KvpoYlBBc7of98gR2LwZvvoKNm60nj+xaRN8/TVUVzsTU2qq/ziI\n77hITs6xZ7V1uj6jidalvXSMQUWMtDQ49VRr8VVXZ3VPNSSKTZusxLFxIxw+HNqYysrgiy+spTER\na3wm0BVYubnW0/cOHLDir621XoNZb3gVsZ4BnpZmLenp1lVfHZH+/ec8bTGoiGCM9cXbkCgaXg8e\ntL5UfReXK/htYA2kb9liJYZwkphoJQjfZBHotWE9MdFKNI2XmprA2wMtyclWl1pGhrX4ricmtu3n\nqa2FoiKri3HXLmtql4Z136WiArKzraQ7YID16rukpNhTv2D9Xh06BNu3w44d3uXbb606HTTI6rIc\nNMhqNcbE2PfZ7UW7kpRqJWOsJNN4EL1h2blT/5JNSWmaLBrWG17T0qwr1wJ9+e/da43vtFXPnv6J\nwjd5ZGT4jxNVV1tf8r5f+o2TQEVFcJ+bkAADB3oTRcProEFWay9chWViEJGJwCNY8zI9ZYz5Q4Bj\n5gKTgHJgujFmbYBjNDHYSPtxW6aqyvpCaTyIvnWr9eVSU2PdFxIba/1VGROD33rjcuN9xlhXeRUX\nW+MvR45YXUwdUwGQ36p3du5sJYr4eOvfZc+e9knoDVfS+SaMAQOsf+fqaqvV1rC0tJyVBVOntj62\nsBtjEBEX8FfgXGA3sEZEXjXGbPI5ZhLQ3xgzUERGA48B3wllXArWrl2riaEFEhLgxBOtJZBHHlnL\nrbfm2/Z5xlhP7mtIEr4Jo7n1ykqIi7O+jBpeW7LExFjdaQcPWt12Bw/6r9fWtv3n6t7d+qLr3dt6\nbbz07g3z56/lu9/NZ8sWK/lu3oxn/XiXQZeUwKeftiym1FSrmyg721pycqxYDh60uiy/+sp6LSpq\n/hx79ljLe++17LODMX582xJDa4R68HkU8I0xZjuAiLwAXAJs8jnmEmAxgDHmIxFJE5FMY8wx/hlU\nWxUXFzsdQlSxuz5FrC+s1FTrS8ppDS2axsnCd/3AAStBZWQ0/bLPyrL+qg7meeUVFcUMGQJDhjTd\nV1dndev5JgvfBOL7GFywxpF69/Z+6fsuDckgLS24y5SLi60k0ZAoGl43bw7tlXR2zkYQrFAnhixg\np0/5W6xkcaxjdrm3aWJQKkyIeAe6+/VzLo6YGOsqsNxcOO88/33GWHfab9litW6ys62EFBdnz2en\np8Po0dbiq7bWask0Thrbtln1FhdndW3FxXkX3/Kx9sXFOVPferlqB1VYWOh0CFFF69M+ra1LEWvG\n38xMe+M5nthYazxhwAC46KL2/exQCengs4h8B7jPGDPRXb4bML4D0CLyGPCeMeZFd3kTML5xV5KI\n6MizUkq1QlgNPgNrgAEikgPsAa4AGg+jvAbcBLzoTiTFgcYXWvqDKaWUap2QJgZjTJ2IzAKW4r1c\ndaOIXG/tNk8YY94UkQtFZDPW5aozQhmTUkqpY4uYG9yUUkq1j4h4VIqITBSRTSLytYjc5XQ8kU5E\nCkXkcxH5TEQ+djqeSCIiT4lIkYh84bOti4gsFZGvRORtEdEnSgSpmfq8V0S+FZFP3ctEJ2OMFCLS\nR0TeFZEvRWSdiNzs3t7i38+wTww+N8ldAAwFporIYGejinj1QL4xZoQxpvHlw+rYFmD9Lvq6G1hu\njBkEvAv8ot2jilyB6hNgjjFmpHv5d3sHFaFqgduNMUOBMcBN7u/KFv9+hn1iwOcmOWNMDdBwk5xq\nPSEy/u3DjjFmFdB4ntdLgEXu9UXApe0aVARrpj7B+h1VLWCM2dswnZAxpgzYCPShFb+fkfDlEOgm\nuTC4FzSiGWCZiKwRkZlOBxMFejRcSWeM2Qv0cDieaDBLRNaKyJPaNddyIpIL5AEfApkt/f2MhMSg\n7HeGMWYkcCFWc/NMpwOKMnpFR9v8HehnjMkD9gJzHI4noohIKvAv4BZ3y6Hx7+Nxfz8jITHsArJ9\nyn3c21QrGWP2uF/3Ay/TdJoS1TJFIpIJICI9gX0OxxPRjDH7faZS/gdwupPxRBIRicVKCk8bY151\nb27x72ckJAbPTXIiEo91k9xrDscUsUQk2f0XBSKSApwPrHc2qogj+PeBvwZMd6//EHi18RvUMfnV\np/vLq8H30d/PlpgPbDDG/MVnW4t/PyPiPgb35Wp/wXuT3IMOhxSxRKQvVivBYN3g+KzWZ/BE5Dms\nhwVkYE30eC/wCrAEOAHYDkwxxuj0tUFopj7PxuofrwcKget1tuXjE5EzgBXAOqz/3wb4JfAx8E9a\n8PsZEYlBKaVU+4mEriSllFLtSBODUkopP5oYlFJK+dHEoJRSyo8mBqWUUn40MSillPKjiUGpdiAi\n40XkdafjUCoYmhiUaj9605CKCJoYlPIhIleJyEfuB8TMExGXiJSKyBwRWS8iy0Qkw31snoj8xz0L\n6EsNs4CKSH/3cWtF5BP33eYAnURkiYhsFJGnHfshlToOTQxKubkfavIDYKx79tl64CogGfjYGDMM\na8qBe91vWQTc6Z4FdL3P9meBR93bxwJ73NvzgJuBIUB/ERkb+p9KqZaLdToApcLIucBIYI2ICJCI\nNX9PPdZcMwDPAC+JSGcgzf2gGbCSxD/dExRmGWNeAzDGVANYp+PjhpltRWQtkAusboefS6kW0cSg\nlJcAi4wxv/LbKDK70XHG5/iWqPJZr0P//6kwpV1JSnm9A1wuIt3B8xD1bCAGuNx9zFXAKmNMCXDI\nPaMlwDTgffeDUXaKyCXuc8SLSFK7/hRKtZH+xaKUmzFmo4j8GlgqIi6gGpgFlAOj3C2HIqxxCLDm\ntn/c/cW/FZjh3j4NeEJEfuM+x+RAHxe6n0SpttFpt5U6DhEpNcZ0cjoOpdqLdiUpdXz615PqULTF\noJRSyo+2GJRSSvnRxKCUUsqPJgallFJ+NDEopZTyo4lBKaWUH00MSiml/Px/iaVSf1zaGtEAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1206ac490>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.plot(train_loss, linewidth=3, label=\"train loss\")\n",
    "pyplot.plot(test_loss, linewidth=3, label=\"valid loss\")\n",
    "pyplot.grid()\n",
    "pyplot.legend()\n",
    "pyplot.ylim(0, 3.0)\n",
    "pyplot.xlabel(\"epoch\")\n",
    "pyplot.ylabel(\"loss\")\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Train loss is far lower than valid loss : we probably overfit, but still valid loss is not so bad ;)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}