pytorch_ipynb/cnn/cnn-alexnet-cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UEBilEjLj5wY"
   },
   "source": [
    "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
    "- Author: Sebastian Raschka\n",
    "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 119
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 536,
     "status": "ok",
     "timestamp": 1524974472601,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "GOzuY8Yvj5wb",
    "outputId": "c19362ce-f87a-4cc2-84cc-8d7b4b9e6007"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sebastian Raschka \n",
      "\n",
      "CPython 3.7.3\n",
      "IPython 7.9.0\n",
      "\n",
      "torch 1.7.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -a 'Sebastian Raschka' -v -p torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rH4XmErYj5wm"
   },
   "source": [
    "# AlexNet CIFAR-10 Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Network Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "References\n",
    "    \n",
    "- [1] Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton. \"[Imagenet classification with deep convolutional neural networks.](https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf)\" In Advances in Neural Information Processing Systems, pp. 1097-1105. 2012.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MkoGLH_Tj5wn"
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "ORj09gnrj5wp"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Subset\n",
    "\n",
    "from torchvision import datasets\n",
    "from torchvision import transforms\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "I6hghKPxj5w0"
   },
   "source": [
    "## Model Settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setting a random seed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I recommend using a function like the following one prior to using dataset loaders and initializing a model if you want to ensure the data is shuffled in the same manner if you rerun this notebook and the model gets the same initial random weights:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_all_seeds(seed):\n",
    "    os.environ[\"PL_GLOBAL_SEED\"] = str(seed)\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Setting cuDNN and PyTorch algorithmic behavior to deterministic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar to the `set_all_seeds` function above, I recommend setting the behavior of PyTorch and cuDNN to deterministic (this is particulary relevant when using GPUs). We can also define a function for that:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_deterministic():\n",
    "    if torch.cuda.is_available():\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "    torch.set_deterministic(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 23936,
     "status": "ok",
     "timestamp": 1524974497505,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "NnT0sZIwj5wu",
    "outputId": "55aed925-d17e-4c6a-8c71-0d9b3bde5637"
   },
   "outputs": [],
   "source": [
    "##########################\n",
    "### SETTINGS\n",
    "##########################\n",
    "\n",
    "# Hyperparameters\n",
    "RANDOM_SEED = 1\n",
    "LEARNING_RATE = 0.0001\n",
    "BATCH_SIZE = 256\n",
    "NUM_EPOCHS = 40\n",
    "\n",
    "# Architecture\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# Other\n",
    "DEVICE = \"cuda:0\"\n",
    "\n",
    "set_all_seeds(RANDOM_SEED)\n",
    "\n",
    "# Deterministic behavior not yet supported by AdaptiveAvgPool2d\n",
    "#set_deterministic()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "sys.path.insert(0, \"..\") # to include ../helper_evaluate.py etc.\n",
    "\n",
    "from helper_evaluate import compute_accuracy\n",
    "from helper_data import get_dataloaders_cifar10\n",
    "from helper_train import train_classifier_simple_v1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "### Set random seed ###\n",
    "set_all_seeds(RANDOM_SEED)\n",
    "\n",
    "##########################\n",
    "### Dataset\n",
    "##########################\n",
    "\n",
    "train_transforms = transforms.Compose([transforms.Resize((70, 70)),\n",
    "                                       transforms.RandomCrop((64, 64)),\n",
    "                                       transforms.ToTensor()])\n",
    "\n",
    "test_transforms = transforms.Compose([transforms.Resize((70, 70)),\n",
    "                                      transforms.CenterCrop((64, 64)),\n",
    "                                      transforms.ToTensor()])\n",
    "\n",
    "\n",
    "train_loader, valid_loader, test_loader = get_dataloaders_cifar10(\n",
    "    batch_size=BATCH_SIZE, \n",
    "    num_workers=2, \n",
    "    train_transforms=train_transforms,\n",
    "    test_transforms=test_transforms,\n",
    "    validation_fraction=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Set:\n",
      "\n",
      "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
      "Image label dimensions: torch.Size([256])\n",
      "tensor([6, 9, 9, 4, 1, 1, 2, 7, 8, 3])\n",
      "\n",
      "Validation Set:\n",
      "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
      "Image label dimensions: torch.Size([256])\n",
      "tensor([7, 1, 4, 1, 0, 2, 2, 5, 9, 6])\n",
      "\n",
      "Testing Set:\n",
      "Image batch dimensions: torch.Size([256, 3, 64, 64])\n",
      "Image label dimensions: torch.Size([256])\n",
      "tensor([6, 9, 9, 4, 1, 1, 2, 7, 8, 3])\n"
     ]
    }
   ],
   "source": [
    "# Checking the dataset\n",
    "print('Training Set:\\n')\n",
    "for images, labels in train_loader:  \n",
    "    print('Image batch dimensions:', images.size())\n",
    "    print('Image label dimensions:', labels.size())\n",
    "    print(labels[:10])\n",
    "    break\n",
    "    \n",
    "# Checking the dataset\n",
    "print('\\nValidation Set:')\n",
    "for images, labels in valid_loader:  \n",
    "    print('Image batch dimensions:', images.size())\n",
    "    print('Image label dimensions:', labels.size())\n",
    "    print(labels[:10])\n",
    "    break\n",
    "\n",
    "# Checking the dataset\n",
    "print('\\nTesting Set:')\n",
    "for images, labels in train_loader:  \n",
    "    print('Image batch dimensions:', images.size())\n",
    "    print('Image label dimensions:', labels.size())\n",
    "    print(labels[:10])\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "##########################\n",
    "### MODEL\n",
    "##########################\n",
    "\n",
    "class AlexNet(nn.Module):\n",
    "\n",
    "    def __init__(self, num_classes):\n",
    "        super(AlexNet, self).__init__()\n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(64, 192, kernel_size=5, padding=2),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "            nn.Conv2d(192, 384, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(256, 256, kernel_size=3, padding=1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(kernel_size=3, stride=2),\n",
    "        )\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(256 * 6 * 6, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(4096, num_classes)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), 256 * 6 * 6)\n",
    "        logits = self.classifier(x)\n",
    "        probas = F.softmax(logits, dim=1)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "_lza9t_uj5w1"
   },
   "outputs": [],
   "source": [
    "torch.manual_seed(RANDOM_SEED)\n",
    "\n",
    "model = AlexNet(NUM_CLASSES)\n",
    "model.to(DEVICE)\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RAodboScj5w6"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1547
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2384585,
     "status": "ok",
     "timestamp": 1524976888520,
     "user": {
      "displayName": "Sebastian Raschka",
      "photoUrl": "//lh6.googleusercontent.com/-cxK6yOSQ6uE/AAAAAAAAAAI/AAAAAAAAIfw/P9ar_CHsKOQ/s50-c-k-no/photo.jpg",
      "userId": "118404394130788869227"
     },
     "user_tz": 240
    },
    "id": "Dzh3ROmRj5w7",
    "outputId": "5f8fd8c9-b076-403a-b0b7-fd2d498b48d7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 001/040 | Batch 0000/0176 | Loss: 2.3030\n",
      "Epoch: 001/040 | Batch 0050/0176 | Loss: 1.9805\n",
      "Epoch: 001/040 | Batch 0100/0176 | Loss: 1.8457\n",
      "Epoch: 001/040 | Batch 0150/0176 | Loss: 1.7629\n",
      "Epoch: 001/040 | Train Acc.: 33.087% |  Loss: 1.697\n",
      "Epoch: 001/040 | Validation Acc.: 34.160% |  Loss: 1.671\n",
      "Time elapsed: 0.44 min\n",
      "Epoch: 002/040 | Batch 0000/0176 | Loss: 1.7060\n",
      "Epoch: 002/040 | Batch 0050/0176 | Loss: 1.5886\n",
      "Epoch: 002/040 | Batch 0100/0176 | Loss: 1.6655\n",
      "Epoch: 002/040 | Batch 0150/0176 | Loss: 1.4965\n",
      "Epoch: 002/040 | Train Acc.: 41.529% |  Loss: 1.521\n",
      "Epoch: 002/040 | Validation Acc.: 41.880% |  Loss: 1.511\n",
      "Time elapsed: 0.88 min\n",
      "Epoch: 003/040 | Batch 0000/0176 | Loss: 1.4985\n",
      "Epoch: 003/040 | Batch 0050/0176 | Loss: 1.4119\n",
      "Epoch: 003/040 | Batch 0100/0176 | Loss: 1.3107\n",
      "Epoch: 003/040 | Batch 0150/0176 | Loss: 1.2836\n",
      "Epoch: 003/040 | Train Acc.: 47.098% |  Loss: 1.406\n",
      "Epoch: 003/040 | Validation Acc.: 47.520% |  Loss: 1.400\n",
      "Time elapsed: 1.32 min\n",
      "Epoch: 004/040 | Batch 0000/0176 | Loss: 1.4325\n",
      "Epoch: 004/040 | Batch 0050/0176 | Loss: 1.2214\n",
      "Epoch: 004/040 | Batch 0100/0176 | Loss: 1.1575\n",
      "Epoch: 004/040 | Batch 0150/0176 | Loss: 1.2118\n",
      "Epoch: 004/040 | Train Acc.: 55.071% |  Loss: 1.224\n",
      "Epoch: 004/040 | Validation Acc.: 54.980% |  Loss: 1.229\n",
      "Time elapsed: 1.77 min\n",
      "Epoch: 005/040 | Batch 0000/0176 | Loss: 1.1903\n",
      "Epoch: 005/040 | Batch 0050/0176 | Loss: 1.1410\n",
      "Epoch: 005/040 | Batch 0100/0176 | Loss: 1.0611\n",
      "Epoch: 005/040 | Batch 0150/0176 | Loss: 1.0817\n",
      "Epoch: 005/040 | Train Acc.: 57.938% |  Loss: 1.144\n",
      "Epoch: 005/040 | Validation Acc.: 57.640% |  Loss: 1.147\n",
      "Time elapsed: 2.21 min\n",
      "Epoch: 006/040 | Batch 0000/0176 | Loss: 1.1344\n",
      "Epoch: 006/040 | Batch 0050/0176 | Loss: 1.0903\n",
      "Epoch: 006/040 | Batch 0100/0176 | Loss: 0.9820\n",
      "Epoch: 006/040 | Batch 0150/0176 | Loss: 1.0716\n",
      "Epoch: 006/040 | Train Acc.: 60.222% |  Loss: 1.104\n",
      "Epoch: 006/040 | Validation Acc.: 58.700% |  Loss: 1.120\n",
      "Time elapsed: 2.65 min\n",
      "Epoch: 007/040 | Batch 0000/0176 | Loss: 1.0856\n",
      "Epoch: 007/040 | Batch 0050/0176 | Loss: 0.9864\n",
      "Epoch: 007/040 | Batch 0100/0176 | Loss: 0.9559\n",
      "Epoch: 007/040 | Batch 0150/0176 | Loss: 0.9611\n",
      "Epoch: 007/040 | Train Acc.: 63.953% |  Loss: 1.008\n",
      "Epoch: 007/040 | Validation Acc.: 61.700% |  Loss: 1.055\n",
      "Time elapsed: 3.09 min\n",
      "Epoch: 008/040 | Batch 0000/0176 | Loss: 0.9608\n",
      "Epoch: 008/040 | Batch 0050/0176 | Loss: 0.9866\n",
      "Epoch: 008/040 | Batch 0100/0176 | Loss: 0.9418\n",
      "Epoch: 008/040 | Batch 0150/0176 | Loss: 0.9364\n",
      "Epoch: 008/040 | Train Acc.: 65.887% |  Loss: 0.946\n",
      "Epoch: 008/040 | Validation Acc.: 63.220% |  Loss: 1.006\n",
      "Time elapsed: 3.53 min\n",
      "Epoch: 009/040 | Batch 0000/0176 | Loss: 0.9465\n",
      "Epoch: 009/040 | Batch 0050/0176 | Loss: 0.9415\n",
      "Epoch: 009/040 | Batch 0100/0176 | Loss: 0.9320\n",
      "Epoch: 009/040 | Batch 0150/0176 | Loss: 0.8411\n",
      "Epoch: 009/040 | Train Acc.: 67.782% |  Loss: 0.908\n",
      "Epoch: 009/040 | Validation Acc.: 64.740% |  Loss: 0.995\n",
      "Time elapsed: 3.97 min\n",
      "Epoch: 010/040 | Batch 0000/0176 | Loss: 0.8873\n",
      "Epoch: 010/040 | Batch 0050/0176 | Loss: 0.9821\n",
      "Epoch: 010/040 | Batch 0100/0176 | Loss: 0.8360\n",
      "Epoch: 010/040 | Batch 0150/0176 | Loss: 0.7949\n",
      "Epoch: 010/040 | Train Acc.: 71.416% |  Loss: 0.806\n",
      "Epoch: 010/040 | Validation Acc.: 67.360% |  Loss: 0.912\n",
      "Time elapsed: 4.41 min\n",
      "Epoch: 011/040 | Batch 0000/0176 | Loss: 0.7469\n",
      "Epoch: 011/040 | Batch 0050/0176 | Loss: 0.8446\n",
      "Epoch: 011/040 | Batch 0100/0176 | Loss: 0.7797\n",
      "Epoch: 011/040 | Batch 0150/0176 | Loss: 0.7813\n",
      "Epoch: 011/040 | Train Acc.: 72.213% |  Loss: 0.779\n",
      "Epoch: 011/040 | Validation Acc.: 67.640% |  Loss: 0.913\n",
      "Time elapsed: 4.85 min\n",
      "Epoch: 012/040 | Batch 0000/0176 | Loss: 0.7076\n",
      "Epoch: 012/040 | Batch 0050/0176 | Loss: 0.8669\n",
      "Epoch: 012/040 | Batch 0100/0176 | Loss: 0.7664\n",
      "Epoch: 012/040 | Batch 0150/0176 | Loss: 0.7205\n",
      "Epoch: 012/040 | Train Acc.: 73.053% |  Loss: 0.762\n",
      "Epoch: 012/040 | Validation Acc.: 67.440% |  Loss: 0.918\n",
      "Time elapsed: 5.29 min\n",
      "Epoch: 013/040 | Batch 0000/0176 | Loss: 0.6941\n",
      "Epoch: 013/040 | Batch 0050/0176 | Loss: 0.7947\n",
      "Epoch: 013/040 | Batch 0100/0176 | Loss: 0.7407\n",
      "Epoch: 013/040 | Batch 0150/0176 | Loss: 0.6998\n",
      "Epoch: 013/040 | Train Acc.: 75.658% |  Loss: 0.686\n",
      "Epoch: 013/040 | Validation Acc.: 69.320% |  Loss: 0.883\n",
      "Time elapsed: 5.74 min\n",
      "Epoch: 014/040 | Batch 0000/0176 | Loss: 0.6040\n",
      "Epoch: 014/040 | Batch 0050/0176 | Loss: 0.7107\n",
      "Epoch: 014/040 | Batch 0100/0176 | Loss: 0.6712\n",
      "Epoch: 014/040 | Batch 0150/0176 | Loss: 0.6610\n",
      "Epoch: 014/040 | Train Acc.: 74.813% |  Loss: 0.725\n",
      "Epoch: 014/040 | Validation Acc.: 68.140% |  Loss: 0.937\n",
      "Time elapsed: 6.18 min\n",
      "Epoch: 015/040 | Batch 0000/0176 | Loss: 0.7104\n",
      "Epoch: 015/040 | Batch 0050/0176 | Loss: 0.6965\n",
      "Epoch: 015/040 | Batch 0100/0176 | Loss: 0.6800\n",
      "Epoch: 015/040 | Batch 0150/0176 | Loss: 0.6326\n",
      "Epoch: 015/040 | Train Acc.: 76.784% |  Loss: 0.660\n",
      "Epoch: 015/040 | Validation Acc.: 69.560% |  Loss: 0.926\n",
      "Time elapsed: 6.62 min\n",
      "Epoch: 016/040 | Batch 0000/0176 | Loss: 0.5889\n",
      "Epoch: 016/040 | Batch 0050/0176 | Loss: 0.6394\n",
      "Epoch: 016/040 | Batch 0100/0176 | Loss: 0.5344\n",
      "Epoch: 016/040 | Batch 0150/0176 | Loss: 0.5194\n",
      "Epoch: 016/040 | Train Acc.: 78.216% |  Loss: 0.620\n",
      "Epoch: 016/040 | Validation Acc.: 69.060% |  Loss: 0.925\n",
      "Time elapsed: 7.06 min\n",
      "Epoch: 017/040 | Batch 0000/0176 | Loss: 0.4660\n",
      "Epoch: 017/040 | Batch 0050/0176 | Loss: 0.6028\n",
      "Epoch: 017/040 | Batch 0100/0176 | Loss: 0.5386\n",
      "Epoch: 017/040 | Batch 0150/0176 | Loss: 0.5476\n",
      "Epoch: 017/040 | Train Acc.: 79.882% |  Loss: 0.563\n",
      "Epoch: 017/040 | Validation Acc.: 70.940% |  Loss: 0.900\n",
      "Time elapsed: 7.50 min\n",
      "Epoch: 018/040 | Batch 0000/0176 | Loss: 0.4768\n",
      "Epoch: 018/040 | Batch 0050/0176 | Loss: 0.6519\n",
      "Epoch: 018/040 | Batch 0100/0176 | Loss: 0.5177\n",
      "Epoch: 018/040 | Batch 0150/0176 | Loss: 0.5190\n",
      "Epoch: 018/040 | Train Acc.: 79.673% |  Loss: 0.575\n",
      "Epoch: 018/040 | Validation Acc.: 69.540% |  Loss: 0.961\n",
      "Time elapsed: 7.94 min\n",
      "Epoch: 019/040 | Batch 0000/0176 | Loss: 0.4714\n",
      "Epoch: 019/040 | Batch 0050/0176 | Loss: 0.5929\n",
      "Epoch: 019/040 | Batch 0100/0176 | Loss: 0.5053\n",
      "Epoch: 019/040 | Batch 0150/0176 | Loss: 0.5581\n",
      "Epoch: 019/040 | Train Acc.: 81.029% |  Loss: 0.535\n",
      "Epoch: 019/040 | Validation Acc.: 70.120% |  Loss: 0.971\n",
      "Time elapsed: 8.38 min\n",
      "Epoch: 020/040 | Batch 0000/0176 | Loss: 0.3960\n",
      "Epoch: 020/040 | Batch 0050/0176 | Loss: 0.5640\n",
      "Epoch: 020/040 | Batch 0100/0176 | Loss: 0.5285\n",
      "Epoch: 020/040 | Batch 0150/0176 | Loss: 0.4373\n",
      "Epoch: 020/040 | Train Acc.: 83.053% |  Loss: 0.497\n",
      "Epoch: 020/040 | Validation Acc.: 71.060% |  Loss: 0.936\n",
      "Time elapsed: 8.82 min\n",
      "Epoch: 021/040 | Batch 0000/0176 | Loss: 0.3562\n",
      "Epoch: 021/040 | Batch 0050/0176 | Loss: 0.4917\n",
      "Epoch: 021/040 | Batch 0100/0176 | Loss: 0.5012\n",
      "Epoch: 021/040 | Batch 0150/0176 | Loss: 0.5080\n",
      "Epoch: 021/040 | Train Acc.: 82.573% |  Loss: 0.489\n",
      "Epoch: 021/040 | Validation Acc.: 70.460% |  Loss: 0.943\n",
      "Time elapsed: 9.26 min\n",
      "Epoch: 022/040 | Batch 0000/0176 | Loss: 0.4068\n",
      "Epoch: 022/040 | Batch 0050/0176 | Loss: 0.4859\n",
      "Epoch: 022/040 | Batch 0100/0176 | Loss: 0.4582\n",
      "Epoch: 022/040 | Batch 0150/0176 | Loss: 0.4259\n",
      "Epoch: 022/040 | Train Acc.: 83.702% |  Loss: 0.468\n",
      "Epoch: 022/040 | Validation Acc.: 71.920% |  Loss: 0.940\n",
      "Time elapsed: 9.70 min\n",
      "Epoch: 023/040 | Batch 0000/0176 | Loss: 0.4236\n",
      "Epoch: 023/040 | Batch 0050/0176 | Loss: 0.4781\n",
      "Epoch: 023/040 | Batch 0100/0176 | Loss: 0.4485\n",
      "Epoch: 023/040 | Batch 0150/0176 | Loss: 0.3734\n",
      "Epoch: 023/040 | Train Acc.: 83.813% |  Loss: 0.461\n",
      "Epoch: 023/040 | Validation Acc.: 70.860% |  Loss: 0.958\n",
      "Time elapsed: 10.14 min\n",
      "Epoch: 024/040 | Batch 0000/0176 | Loss: 0.3325\n",
      "Epoch: 024/040 | Batch 0050/0176 | Loss: 0.3642\n",
      "Epoch: 024/040 | Batch 0100/0176 | Loss: 0.3749\n",
      "Epoch: 024/040 | Batch 0150/0176 | Loss: 0.3309\n",
      "Epoch: 024/040 | Train Acc.: 85.018% |  Loss: 0.425\n",
      "Epoch: 024/040 | Validation Acc.: 71.560% |  Loss: 0.965\n",
      "Time elapsed: 10.58 min\n",
      "Epoch: 025/040 | Batch 0000/0176 | Loss: 0.3156\n",
      "Epoch: 025/040 | Batch 0050/0176 | Loss: 0.3344\n",
      "Epoch: 025/040 | Batch 0100/0176 | Loss: 0.3945\n",
      "Epoch: 025/040 | Batch 0150/0176 | Loss: 0.3581\n",
      "Epoch: 025/040 | Train Acc.: 84.291% |  Loss: 0.450\n",
      "Epoch: 025/040 | Validation Acc.: 69.320% |  Loss: 1.077\n",
      "Time elapsed: 11.02 min\n",
      "Epoch: 026/040 | Batch 0000/0176 | Loss: 0.3812\n",
      "Epoch: 026/040 | Batch 0050/0176 | Loss: 0.3231\n",
      "Epoch: 026/040 | Batch 0100/0176 | Loss: 0.4320\n",
      "Epoch: 026/040 | Batch 0150/0176 | Loss: 0.3551\n",
      "Epoch: 026/040 | Train Acc.: 83.344% |  Loss: 0.467\n",
      "Epoch: 026/040 | Validation Acc.: 69.180% |  Loss: 1.108\n",
      "Time elapsed: 11.45 min\n",
      "Epoch: 027/040 | Batch 0000/0176 | Loss: 0.3591\n",
      "Epoch: 027/040 | Batch 0050/0176 | Loss: 0.3114\n",
      "Epoch: 027/040 | Batch 0100/0176 | Loss: 0.2481\n",
      "Epoch: 027/040 | Batch 0150/0176 | Loss: 0.3428\n",
      "Epoch: 027/040 | Train Acc.: 84.931% |  Loss: 0.433\n",
      "Epoch: 027/040 | Validation Acc.: 69.600% |  Loss: 1.126\n",
      "Time elapsed: 11.90 min\n",
      "Epoch: 028/040 | Batch 0000/0176 | Loss: 0.3464\n",
      "Epoch: 028/040 | Batch 0050/0176 | Loss: 0.2866\n",
      "Epoch: 028/040 | Batch 0100/0176 | Loss: 0.2776\n",
      "Epoch: 028/040 | Batch 0150/0176 | Loss: 0.3536\n",
      "Epoch: 028/040 | Train Acc.: 86.289% |  Loss: 0.399\n",
      "Epoch: 028/040 | Validation Acc.: 70.460% |  Loss: 1.114\n",
      "Time elapsed: 12.34 min\n",
      "Epoch: 029/040 | Batch 0000/0176 | Loss: 0.3019\n",
      "Epoch: 029/040 | Batch 0050/0176 | Loss: 0.3655\n",
      "Epoch: 029/040 | Batch 0100/0176 | Loss: 0.2486\n",
      "Epoch: 029/040 | Batch 0150/0176 | Loss: 0.4025\n",
      "Epoch: 029/040 | Train Acc.: 89.231% |  Loss: 0.308\n",
      "Epoch: 029/040 | Validation Acc.: 71.820% |  Loss: 1.050\n",
      "Time elapsed: 12.78 min\n",
      "Epoch: 030/040 | Batch 0000/0176 | Loss: 0.2827\n",
      "Epoch: 030/040 | Batch 0050/0176 | Loss: 0.3269\n",
      "Epoch: 030/040 | Batch 0100/0176 | Loss: 0.2699\n",
      "Epoch: 030/040 | Batch 0150/0176 | Loss: 0.3385\n",
      "Epoch: 030/040 | Train Acc.: 89.320% |  Loss: 0.308\n",
      "Epoch: 030/040 | Validation Acc.: 71.940% |  Loss: 1.063\n",
      "Time elapsed: 13.22 min\n",
      "Epoch: 031/040 | Batch 0000/0176 | Loss: 0.2108\n",
      "Epoch: 031/040 | Batch 0050/0176 | Loss: 0.2322\n",
      "Epoch: 031/040 | Batch 0100/0176 | Loss: 0.2662\n",
      "Epoch: 031/040 | Batch 0150/0176 | Loss: 0.1860\n",
      "Epoch: 031/040 | Train Acc.: 88.807% |  Loss: 0.316\n",
      "Epoch: 031/040 | Validation Acc.: 71.720% |  Loss: 1.073\n",
      "Time elapsed: 13.66 min\n",
      "Epoch: 032/040 | Batch 0000/0176 | Loss: 0.2714\n",
      "Epoch: 032/040 | Batch 0050/0176 | Loss: 0.3018\n",
      "Epoch: 032/040 | Batch 0100/0176 | Loss: 0.2323\n",
      "Epoch: 032/040 | Batch 0150/0176 | Loss: 0.2668\n",
      "Epoch: 032/040 | Train Acc.: 91.173% |  Loss: 0.249\n",
      "Epoch: 032/040 | Validation Acc.: 71.980% |  Loss: 1.025\n",
      "Time elapsed: 14.10 min\n",
      "Epoch: 033/040 | Batch 0000/0176 | Loss: 0.1913\n",
      "Epoch: 033/040 | Batch 0050/0176 | Loss: 0.3458\n",
      "Epoch: 033/040 | Batch 0100/0176 | Loss: 0.1746\n",
      "Epoch: 033/040 | Batch 0150/0176 | Loss: 0.2072\n",
      "Epoch: 033/040 | Train Acc.: 91.171% |  Loss: 0.245\n",
      "Epoch: 033/040 | Validation Acc.: 72.680% |  Loss: 1.060\n",
      "Time elapsed: 14.54 min\n",
      "Epoch: 034/040 | Batch 0000/0176 | Loss: 0.2103\n",
      "Epoch: 034/040 | Batch 0050/0176 | Loss: 0.2894\n",
      "Epoch: 034/040 | Batch 0100/0176 | Loss: 0.2032\n",
      "Epoch: 034/040 | Batch 0150/0176 | Loss: 0.2118\n",
      "Epoch: 034/040 | Train Acc.: 91.756% |  Loss: 0.239\n",
      "Epoch: 034/040 | Validation Acc.: 73.200% |  Loss: 1.038\n",
      "Time elapsed: 14.98 min\n",
      "Epoch: 035/040 | Batch 0000/0176 | Loss: 0.2010\n",
      "Epoch: 035/040 | Batch 0050/0176 | Loss: 0.2106\n",
      "Epoch: 035/040 | Batch 0100/0176 | Loss: 0.1802\n",
      "Epoch: 035/040 | Batch 0150/0176 | Loss: 0.2216\n",
      "Epoch: 035/040 | Train Acc.: 91.060% |  Loss: 0.264\n",
      "Epoch: 035/040 | Validation Acc.: 71.280% |  Loss: 1.146\n",
      "Time elapsed: 15.42 min\n",
      "Epoch: 036/040 | Batch 0000/0176 | Loss: 0.1478\n",
      "Epoch: 036/040 | Batch 0050/0176 | Loss: 0.1735\n",
      "Epoch: 036/040 | Batch 0100/0176 | Loss: 0.1186\n",
      "Epoch: 036/040 | Batch 0150/0176 | Loss: 0.1835\n",
      "Epoch: 036/040 | Train Acc.: 91.376% |  Loss: 0.254\n",
      "Epoch: 036/040 | Validation Acc.: 72.160% |  Loss: 1.181\n",
      "Time elapsed: 15.86 min\n",
      "Epoch: 037/040 | Batch 0000/0176 | Loss: 0.1154\n",
      "Epoch: 037/040 | Batch 0050/0176 | Loss: 0.1817\n",
      "Epoch: 037/040 | Batch 0100/0176 | Loss: 0.1166\n",
      "Epoch: 037/040 | Batch 0150/0176 | Loss: 0.1973\n",
      "Epoch: 037/040 | Train Acc.: 91.278% |  Loss: 0.259\n",
      "Epoch: 037/040 | Validation Acc.: 71.480% |  Loss: 1.241\n",
      "Time elapsed: 16.31 min\n",
      "Epoch: 038/040 | Batch 0000/0176 | Loss: 0.1402\n",
      "Epoch: 038/040 | Batch 0050/0176 | Loss: 0.1672\n",
      "Epoch: 038/040 | Batch 0100/0176 | Loss: 0.1366\n",
      "Epoch: 038/040 | Batch 0150/0176 | Loss: 0.1037\n",
      "Epoch: 038/040 | Train Acc.: 91.447% |  Loss: 0.256\n",
      "Epoch: 038/040 | Validation Acc.: 71.260% |  Loss: 1.256\n",
      "Time elapsed: 16.75 min\n",
      "Epoch: 039/040 | Batch 0000/0176 | Loss: 0.1505\n",
      "Epoch: 039/040 | Batch 0050/0176 | Loss: 0.1537\n",
      "Epoch: 039/040 | Batch 0100/0176 | Loss: 0.1592\n",
      "Epoch: 039/040 | Batch 0150/0176 | Loss: 0.1947\n",
      "Epoch: 039/040 | Train Acc.: 92.571% |  Loss: 0.223\n",
      "Epoch: 039/040 | Validation Acc.: 72.040% |  Loss: 1.266\n",
      "Time elapsed: 17.19 min\n",
      "Epoch: 040/040 | Batch 0000/0176 | Loss: 0.1707\n",
      "Epoch: 040/040 | Batch 0050/0176 | Loss: 0.1500\n",
      "Epoch: 040/040 | Batch 0100/0176 | Loss: 0.1422\n",
      "Epoch: 040/040 | Batch 0150/0176 | Loss: 0.1099\n",
      "Epoch: 040/040 | Train Acc.: 95.191% |  Loss: 0.139\n",
      "Epoch: 040/040 | Validation Acc.: 72.740% |  Loss: 1.201\n",
      "Time elapsed: 17.63 min\n",
      "Total Training Time: 17.63 min\n"
     ]
    }
   ],
   "source": [
    "log_dict = train_classifier_simple_v1(num_epochs=NUM_EPOCHS, model=model, \n",
    "                                      optimizer=optimizer, device=DEVICE, \n",
    "                                      train_loader=train_loader, valid_loader=valid_loader, \n",
    "                                      logging_interval=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'cost_list' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-550123761591>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'train_loss_per_batch'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Minibatch cost'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m plt.plot(np.convolve(cost_list, \n\u001b[0m\u001b[1;32m      3\u001b[0m                      np.ones(200,)/200, mode='valid'), \n\u001b[1;32m      4\u001b[0m          label='Running average')\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'cost_list' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3hUVfoH8O+bRggEEkjogVAivYXQQYogAiqiWHfVdXVde3fF3pG1/xQV0bUrVhSlSFdQIhBaaKEHCC2BQAIppJ3fH1My5U5JMjN3yvfzPHmYuffMvW8U3pyce857RCkFIiIKfGF6B0BERJ7BhE5EFCSY0ImIggQTOhFRkGBCJyIKEhF63TghIUElJyfrdXsiooC0fv36E0qpRK1zuiX05ORkZGRk6HV7IqKAJCIHHJ3jkAsRUZBgQiciChJM6EREQYIJnYgoSDChExEFCSZ0IqIgwYRORBQkAjKhPz13K/LOnNM7DCIivxJwCX3G8t34NP0A+r+4VO9QiIj8SsAl9OsHJwMAhqck6BsIEZGfCbiE3rh+JJKa1EfTBlF6h0JE5FcCLqEDQKPoSBSWVugdBhGRXwnIhN64fiQKSsr1DoOIyK8EbEIvZEInIrISkAm9skphd+5ZvcMgIvIrAZnQF28/DgAoOsdxdCIik4BM6CYVlUrvEIiI/EZAJ/SzZeyhExGZBHRC/yw9W+8QiIj8RkAn9LOci05EZBaQCb1Fo2gAwA8bcnSOhIjIfwRkQo+MEABAaXmVzpEQEfmPgEzoUeEBGTYRkVcFZGac2LOl3iEQEfmdgEzoV6Yl6R0CEZHfCciEHh4meodAROR3AjKhRzChExHZCciEHmaR0F+Ytx3HC0t1jIaIyD8EZEK37KF/+Md+PPTdZh2jISLyDwGZ0G3H0EvLK3WKhIjIfwRkQo8Isw67nFUXiYgCM6Hb5HOUV3LFKBGRy4QuIkkiskJEdojINhG5V6ONiMhbIrJHRDJFJNU74RrY9tArq9hDJyKKcKNNBYAHlVIbRCQWwHoRWaKU2m7RZjyAFOPXQADvGf/0CttZi0zoRERu9NCVUkeVUhuMr88A2AGgtU2zSQA+UwZ/AYgTEa+tzxexzuhM6ERENRxDF5FkAH0BrLE51RrAIYv3ObBP+hCRW0UkQ0Qy8vLyahapE5WKCZ2IyO2ELiINAfwA4D6lVKHtaY2P2GVZpdQspVSaUiotMTGxZpE6ceBksceuRUQUqNxK6CISCUMy/1IpNUejSQ4Ay4pZbQAcqXt4jvVq09iblyciCjjuzHIRAP8DsEMp9bqDZj8DuME422UQgAKl1FEPxmmnR2smdCIiS+700IcCuB7AaBHZZPyaICK3ichtxjYLAOwDsAfABwDu8E641cb3aOHtWxARBRSX0xaVUn9Ae4zcso0CcKengnLH8BTrMfj1B/LRr10TX4ZARORXAnKlqJYr3kvXOwQiIl0FTUIHgHmZXn0OS0Tk1wI6oT97aXer9y/M26FTJERE+gvohN6lRazV+2OFpfj8rwM6RUNEpK+ATugDOzS1O/bkT1t1iISISH8BndCJiKhaUCb033d5rk4MEVGgCMqE/nl6tt4hEBH5XMAn9Ml97Yo6gsUXiSgUBXxCjwy3X8SadeyMDpEQEekr4BN6RLj9t3D4dIkOkRAR6SvgE3qURkInIgpFAZ8NI2w3GDVSSiEjOx+KA+pEFCICPqGP76ldRnfWyn2YMjMdv2R6tSw7EZHfCPiE7qhk7ksLswAAO47a7pZHRBScAj6hA8ALl/VweO5ceRU++mM/yiqqfBgREZHvudzgIhA4245u9tqDKCmvRHFZBe4aneLDqIiIfCsoeuhdW8Y6PFdSXgkAeHXxLl+FQ0Ski6BI6PUiwvUOgYhId0GR0ImIiAmdiChoMKETEQWJkErov+3M1TsEIiKvCamE/t36HL1DICLymqBJ6M9e2t1lm/mZR5F9osgH0RAR+V7QJPToSPe+lZGv/ubdQIiIdBI0Cb1DYkMAwCW9W+kcCRGRPoJi6T8A9E9uguUPjkB8TBR+2XxE73CIiHwuaHrogKGXHt8gymW7S2f8gdzCUh9ERETkO0GV0N2VmVOAORsP6x0GEZFHhWRCN5m2YAfumb1R7zCIiDwiKBP62G7NXbY5eroEs1buw88cbyeiIBGyCf3T9AM+iISIyHeCMqFflZZUo/b78s5i6fbjXoqGiMg3gjKhA8BNQ5MxsnOiW21Hv/Y7bvksAwCQe6YUh/KLvRkaEZFXuEzoIvKRiOSKyFYH50eKSIGIbDJ+PeX5MGvu6Uu6472/9avx5wa8uAzDX17hhYiIiLzLnYVFnwCYAeAzJ21WKaUu9khEHlQ/ijsZEVHocNlDV0qtBJDvg1iIiKgOPDWGPlhENovIQhFxWPZQRG4VkQwRycjLy/PQrT3nVFGZ3iEQEdWaJxL6BgDtlFK9AbwN4CdHDZVSs5RSaUqptMRE9x5Y+tLjP23ROwQiolqrc0JXShUqpc4aXy8AECkiCXWOzEM+vqk/npjY1a22C7Yc83I0RETeU+eELiItRESMrwcYr3myrtf1lFGdm+GW4R30DoOIyOtcznIRkdkARgJIEJEcAE8DiAQApdRMAFMA3C4iFQBKAFyjlFJei5iIiDS5TOhKqWtdnJ8Bw7TGoLJwy1FkHDiF+8akIDY6Uu9wiIhcCpoNLjzt9i83ADDMfHn96j7m44dPlyChYRTqRXCOOxH5l6Bd+u8plnXTS8srMXT6cjz0XaZm29PFZfiJddaJSCfsoddAeWUVAGBFVq7m+bu+2og/9pxAatt4tG0a48vQiIhCp4f+/KTuGJ7imdmUjp75HjldAgAoMyZ+IiJfCpmEfv3gZHx+88A6XcM4OxOOpvAoc7s63YaIqFZCJqHXRUZ2PpKnzseBk0UAgOKySs1euukY8zkR6YEJ3Q1TZqYDABZtrV5Jes/Xm+zaVffQmdKJyPdCLqFP7Nmy1p+ttOiV/6KxF6npNNM5Eekh5BL6tMt71vqzVS7WvypjH50ddCLSQ8gl9Mb1a7/qs8pJRYPTxWU4lG+Y5SLsoxORDkIuoVta+/gFNWr//u/7HJ574NvN5tfsoRORHkI6oTeLjUbruPq1/vz5L69AlXEc5lhBqfk4EzoR6SEkV4q+cFkPlJZXAgD+nDoayVPn1+o6B/OLUVhajs05BQgPq87inOVCRHoIyYT+90HtrN4P7dQUf+6pXQn3u2dvxKrdJ6yOMZ0TkR5CesjFJKwOPWrbZA44HnL5/K8DWLFTuw4MEVFdMaED6JsU59HrOZrl8uRPW3HTx+s8ei8iIhMmdAD3jjnP49c8efacx69JROQMEzpg9UDTE/5v2S70e2EpNhw8ZT5W5WpVEhFRHTGhe8HstYcAANsOF5iPdXhsgV7hEFGIYEL3oifnbsNzv2zXOwwiChFM6F720Z/77Y7lFpYiM+e008+9vWw3Br+0zFthEVEQCsl56L5mWsRkMmCaIVHPuWMIUtvGa37mtSW7vB4XEQUX9tB9oMuTv2oe336k0MeREFEwY0K3MSC5CQAgJirc6/fKO1M9tbG4rAJP/LQFZ89VeP2+RBScOORitOKhkcg+WYRRnZvh+/U5uLB7c/R6ZrFX77n/RJH59efpB/DFXwdxqqgcj1zUxav3JaLgxIRu1D6hAdonNAAATOnXBoCht742O99r97Ssr24qFzB/y1HM33LUa/ckouDFIRcnpqS18er1yyurABgemp4uLvfqvYgo+DGhO3FlP+8m9OKySmw4eApdnvwV7/6216v3IqLgxyEXJ7xd13zV7hOa1RqJiGqDPfQA8czP23DBa7/pHQYR+TEm9Fp472+p6J+svSDIG577ZTs+WZ2NvXlF+O+vWThVVOazexNR4GBCd6F3m8Z2x8b3bInvbhvisxgsywe899tePDl3q8/uTUSBgwndhR/vGIp90ybodv/J7/5pd2xe5lHka/TSdx47gzOlnC1DFKqY0F0ICxOEWdRLf3hcZ/Prey9I8fr9Nx7ULuI19YdM8+tdx8/gyOkSjHtzJdJeWOr1mIjIPzGhu+md61IBAFdazE2/f+x5uLhXS13iWbz9OB753pDUL3xjJYZMXw4AOFdRpUs8RKQ/lwldRD4SkVwR0Ry4FYO3RGSPiGSKSKrnw9TfxF4tkT19IprFRlsdv8LLc9Wd+SbjUI3a5xeV2VV+JKLg4U4P/RMAFzk5Px5AivHrVgDv1T0sctfn6dlut019fglu+Git12IhIn25TOhKqZUAnBU0mQTgM2XwF4A4EdFnHEIHbeLqm1//3zV9fH7/J+duq1H7tfvdq03T8+lFmLF8d21CIiKdeGIMvTUAy9/9c4zHQkJK81i0bRIDAGjaoB4+vqm/+b1eNhw8hV7PLMLBk8W1vsaZcxV4dTE32SAKJJ5I6Frr4zW3uBeRW0UkQ0Qy8vLyPHBr/5DUpLqXPqpzM6Q0a6hjNMA9szeisLQC57+yAr/tzPXINfOLylBRyQeuRP7MEwk9B0CSxfs2AI5oNVRKzVJKpSml0hITEz1wa//UqH6krvfPOVVifv3Rn9lYfyAfBSXa89P35J7FhoOnkHOqGLd8ug7FZfYbbJRXViH1+SWYOmeL12ImorrzREL/GcANxtkugwAUKKVCuqD3M5d21zsEs5W78nDFe+kY9epvmufHvP47Ln93NV5amIWlO3KxdId1j3713hPmUgPzMjV/ThORn3Bn2uJsAOkAOotIjojcLCK3ichtxiYLAOwDsAfABwDu8Fq0fk4ZR5oa14/EnhfH253/+6C2vg7JTGtlqSVHdSWv+2AN7pq9sVb3rKxSnCZJ5EMuy+cqpa51cV4BuNNjEQUg0UiHEeFhyHzmQhw9XYpxb64EALRoFG3XTk/fWsxjn5dp+KVKKfvHH7uPn6nV9f/1WQaWZ+Uie/rE2gVIRDXCeuhe1Cg6Eo1aVI+n/3tER7SJj8GkPq3Q/tEFOkZmmJPuqtduElbLuvDLszzzQJaI3MOE7kOR4WG4rK/+MzpHvrLCYTJXCtibd9bqmJf3+SAiD2EtFw94bEJX9E6KQ7927tdIn6hTDRgAyHYyP33BlqMos6kH4+2dm4jIM9hD94BurRph7p1Da/SZd65LxdSLijH85RVeiqp2Fm8/jiSbhVHhNUzoq/ecQNax6nH3BVuOIjY6AsNTgneqKpE/YA9dR7aJ019sySmwem/K56Xlhp7777vykDx1PtZla5cRuO7DNXhu3nbz+zu+3IDr/8caMkTexoROdrJPFlm9t+yfK6Vwo7HA15Uz030YFRG5wiEXH3B3/9FlD47A2dIKLNhyFO+v3OflqBzLPXPO6v2RglLz64oqzaoO+GXzEYzsnIjYaH1XyRKFMiZ0H3B3/9GOiYYaMD1bN8bB/GIs3HrMm2HVylvL7CswZh0rxN2zN2JCzxa4fUQnHaIiIoBDLrr7+B/97Y6FhQmendTd7Z69L729fI/dseIyw2rQBVuOYf0Bx+V5tRYtEZHnMKHrLLWtIWnfNqKj1fFmsdGaPfvY6MD9pYr5nMi7Ajc7BInGMZFuL41f+sAIJDWpj85P/OrlqLwjv7gMCQ3roaSsElnHCtG3rf/9BkIUyNhDDyCdmjVEvYhwv6qNUlpeib251StLz5Tal981ufTtPwAAXZ/6FZPfXY08m4evRFQ3TOhB5Lzmvt9Y46HvNuPh7zPN7z/764DDtpazZQBg7qbDmu3mZR7B/hNFmueIyDEm9CCSltzE5/c0VWk0cVUu96m5W82v/9xzAiVllXjgm004lG8oR7Bwy1Hc9dVGXPDabx6PlSjYMaEHqP9e0dPqff3IcDx8YWdcnZbk4BO+4WzIBQA+S6/uwa/eexJdn/oVczYexvCXV+C7jEO4/csNAADTdPdTRWXYk1tdRqCisgr3fr0RZ885vw9RKGJCD1BX92+LqeO7mN+/OLkH4htE4V/nt9cxqpo5Z1METGtK5OR3/8SY1w315KuqFMa+sRJzNx3BNbO0V6mu3nMCK3cFz361RDXBhB7AbKc6AkCnZrHInj4R1w9qp0NEnmeqDFlQUo43l+4yj61vO1KIXs8swi+brbfFu+7DNbjhI9aNodDEhO7nvr51kFvt2toU+npukv/sa1pbluPxvZ9djHlbqsfrlQIKSytw9+yN2HTotFvXm7Mhh/uiUlBjQvdzgzo0xZtX98Gv9w3XPP/n1NGYc8cQuweiIoLL+rSyOqbnnqa18dicLVbvHe2cdNk7f7p1vQe+3Yy7vqrd/qhEgYAJPQBc1rc1urRopHmudVx982pTW7YbU7xwWU/Ndv7iYL71xhtzNlpPa9yTa72TEhFZ40rRIGZK52O6NkPXlto/EELZ7LUHMapzM7Ro7F+bdxPVFnvoQWxwx6YAgHsuSMGDF3YGYCgfQEDemXN4dM4W3PTJOodtPvpjPw6fLvFhVER1w4QexK5MS8Laxy9ArzZx5mOdmvl+Nak/qjROdD95Vrv8wPHCUjw3bztu+pgzZihwMKEHuWax9sMJDaLCdYjE+66d9ZfdsYzsfBSUlNsdDzOOR9lu5mFiSviuFkoR+RMm9BDUtmkDu2PDOiXoEIlnpe87afX+zi83YMrMdNxsM6xSXlllva+eBlb6pUDEhB6CPr95AGZd38/8vmvLRvjiloFIadYQibH1rNrWi6j+K/LKlF7o0TowHq6uyMrFfOO89e1HC63OTVuwA6KR0X/cmIOjBYYx8x835AAASoxz4c+eq8CHq/ahymYLPqUUzp6rwJjXf8eVM1d7/Psgqgkm9BCU0LAeLuzewvz+uoGG+elLHhiBdY+PQeu4+uZz7RMaoHPzWABAzzaN0SnRP8fgz5RaD6tYPuw07ahk8snqbLvPbz9SiPu/2YzBLy0HALy6eBcA4HSx4bpPz92GF+bvwI82Uyk/Sz+AHk8vwp7cs1iXfarO3wdRXTChk12ZgBUPjcSPd2jvg1pe6Z+DET2fWez0vGWBL6UAyyn6uWdKcdfsDU4//4Oxx/7gd5sBAGv35+OdFXuwaJv1vq8nz57Dh6v2IXnqfBSXcfydfIvz0MlOVEQY6kVoPzi9PLU15m85iujIMJSWV2m28UeP2qw6TXthqfn1wGnLrLbH+9ympntFpfX3eaqoDFe9bygOltzUuuTCU3O3YeNBQ089v6gMMVGGf2J7886i6FyF1YwjIk9jDz2EpbaNsxszN+nSIhY3DG6H9/7ez+r4iPMSMaZrc3z778EID3PxZNGPOBsOsd3r9Mmftlq9L7Gp8d7vhSXm17YVI+db1JvJzCkwv77gtd9x6Qz3ShQQ1RYTegibc8dQrHt8jOa5sDDBc5N6oH1CA3RINMyKiYmMQER4GD68MQ292sRh27PjzO2jIsKQ9fxFdte5dkBbbH9unN3xQDLgxWVW7y2fizrb+PqOL50P4xB5GhM6ufTqlb3xyU390dZmeCE6MtwqiUdH2g/TtG0SYx52CFS2PXRLxwpLHZ4DgMOnS3DLp45XoxJ5EhM6udSgXgRGdm6mec407DKqc6LV8X8MSfZ2WH7LstM+dPpyLN2Ra35vu0VfaXmlYV48kQcEdteJdBcZHoZV/xllP389MnT7CkcLHPfap/6Qibmbj+DuUZ3wwIWd0eXJXwEA2dMn+io8CmKh+6+OPCapSYzdcMuU1DYAgPE9DPPd37i6t8/j8keZhwugFPCWxnZ7JiuycvHlmgMOzxM54lZCF5GLRGSniOwRkaka50eKSIGIbDJ+PeX5UCmQpDQ3bIWXnGB4oDq5bxtkT5+Iz28eYNXu4XGd9QhPN/vyisyvC0vta8wAhkVRj/+4VfMckTMuh1xEJBzAOwDGAsgBsE5EflZKbbdpukopdbEXYqQg4mijjlD0xV/u98ILSsox7o2VOFZYqjk8c7ywFFVKoWXj+hqfplDhTg99AIA9Sql9SqkyAF8DmOTdsChQzbljCN79W6rD87a7yF3Uo4V2Q6PBHZpi/0sTHJ6fNtm/d2FyxnLK4768s3YLmJRFg0e+zzTPqDmUX4wym/nvA6ctw+CXliO/qAzX/28NHvpuM85VVGLupsNW17G0IisXH6zc56Hvxl5VlcKHq/bZPQgm73EnobcGcMjifY7xmK3BIrJZRBaKSODvUEy1kto2HhN6tnR4vmmDKFw7oHpv046JDe16nKbaMgDQpWWs3VZ6lhrXj6xDtP5j9Gu/Y/K7q3HLpxnmYwOnVc9/P1lUXeZ3+Msr7Fa+mjzw7Sas2n0C36/PwVUz03Hv15uwYmeuZtubPlmHFxfs8NB3YO+XzCN4Yf4OvLZ4p9fuQdbcSeha/5psf+RvANBOKdUbwNsAftK8kMitIpIhIhl5eXk1i5SCgojgpct74s+po602vv7h9uraMU1iosyvHW0MbZLaLnCX0r+yyDrRbTlcgKU7jpvf5545hxVZubj47VWosKnyuHK39r+fU0Vl5teHThkqR+YXaY/VO/p89okil+0qqxQe+T4T+/Ic7/NqKorGmvK+405CzwGQZPG+DYAjlg2UUoVKqbPG1wsARIqIXYFtpdQspVSaUiotMTHR9jSFkNZx9a3G0/u1i8fKh0fh3gtScP3g6mJh94xOcXodZys1g8FNn6zD1sOFdpt0KAXc/sV6/LTxMA5ZbK692aLcgKkyQ5XxP9KmQ6eR+vwSnC4ugyN9n1+Cka/+5jKubUcK8E3GIdz79SaHbYL9/40/ciehrwOQIiLtRSQKwDUAfrZsICItxPh7sYgMMF73pN2ViJxo2zQG9489z6pX3jjGMKSy8N7q3vzlqdUjflUhkjUsZ8eYLNx6DPd9swnDX16h+RnTUJWphvuM5XuQX1SGtfvzrdpZ/kBw5VB+MbYdKTD/P3Lnv7/WL1mpzy/BSwu9N9wTqlzOclFKVYjIXQAWAQgH8JFSapuI3GY8PxPAFAC3i0gFgBIA1yhHT2KIaqFry0bIfOZCRIQJYqIiMGeDoS55vMXwTCg54WAvVEvh5qRrfXzr4QJEWmxcUlBSbvUruDOmHx7z7h6meW0tlTaNyiqqkF9Uhvd/34dHx3d1887kDrdWihqHURbYHJtp8XoGgBmeDY1CXUJD62TdKLr6AejCe4fjUH4xGtTjYmdHTLNiKs19K8OftouabBOuO0w9dHf6bd9m5ODlKdULy2avPei0/Yer9mFk52bc0LwWuFKU/E6TBlEY0rEp3rqmr8M2XVs2stp1SUt9jWJhlpo5KB1s8tW/Bjo9Hyhm/rYXyVPnW9WUsbQ7V/vBZkFJOX7ZfETznKmGj6Mhl53HziAz57TmuU81dowyKa+swgvzd2Dyuyw1XBvs3pDfCQ8TfPWvQXW+zh0jO2JzTgESY6Mwe+0hu/NdWjZCy8ZlVg8SLTUI8CqRJodPlzg9/9B3mzGlXxu7B6/3f7MJy7Ny0aN1Y7RPsN5YXMwPXLWvOe7NlbWK1fTzoaSMc9drgz10Cnibn77Q6kHpFzcPxLTJPfHvER3x4Y1peOnyXpqfmza5B+beNQwTHcybjwgXXN5Xa8lF8DlVVIbHLOa2l5ZX4rBx2mP6Xvv5DbYzaHILS5E8dT7eXLrL5b3cGeBxMVuVHGBCp4DXuH4knr6kei3bsJQEXDewLaIinP/1bhNvqO/+r/M7aJ7v1rIRLundynOB+rFKpbAsq3oO/IPfbTYn1cd+1F7EBFT3qN9cttvw59Ldmu1u/GgtkqfOBwDst5jnbpqB0/e5xfjnJ+ugjOm+vFLhiMZvFlN/yESf55zvHxvKmNApqMRGOx8m6dIi1u5YnyTtxUkigsEdmzq8luViKABo5OLe/qysospqj9j5mUeRdeyMVRvLoZvP0w11aEzJ+as11Q86v1ln/9Dz913aC6H++2sWAOBUcTmWZ+VazV2fqrEa9ut1h3C62PlCqS05BVYLrEIJEzoFBVNvsqGLWS+dmjXEred3wMqHR2mer2fTq9fahcmkReNoq/d3jOrkRqT+acj05U7P931uMYZatPk0vbqwmO1Ml0d+cNyjP2kz3XJe5lGH4+XORl122vywsXTJjD/Q9/klDs8Hs8DtUhBZaBQdiScmdsXYbs1dtn1sgv3c5wfHnodurRpBKeCWzzI0PmWvlU1CbxMfvJUOTznpFTuqFaPF9sErAAycttT82vJng6NePWB46MpNQeyxh05B45bhHdCuaQOnbWKitHvcd1+Qggu6NkevpMZu38+yaFj29Ikhu9R900Ht6YnuCAsDCi1qvSiNR6ZKKc357loPa8+es64bk1tYqjkWH6yY0Ckk/PHIKMTHROK5ST2ctktsaJib7mjmy/onxli9T390NObfY1g12dlifP7Gwe2w5P7zcXGvllj1n1HokOD8B00gc7b7kq3ySuvEfCjfOtkWnbMffmn/6AJcPesvu+PXfmB/bNHWY1bvB0xb5nI4KZgwoVNIaBMfg41PXeh0TBww9Lqzp0/EOxo13bOnT0TThtaLkVo2ro/urQy9+vOaVyf0Li0bIaV5LGZcl4qkJjFY/tBIXGHcli+UuZqf/sEq6/rspp752v35bm2mXddfkpRSeGruVmw4eKqOV9IHEzpRDX1wQxqWPjCixp977areSGsX74WIgkduofUG2/1frB5fP+Zk822TgzUoNKalvFLhs/QDuGpmuub5p+duxZUzV9fpHt7EhE5UQ2O7NXdZZ8Sy7oylj27qr3n8038OwID2TeocW6A7cto6aZ84Wz39cMQr2lUlTdJeWIK3lmnPgzf9oDhwsgiT3/1T8+GsOz5NP4B12Ybe++d/HcCaff5VVJazXIg86Kc7hyIjOx8TemrXmWkUHWnYPNu4yMYkIkwwrFOCXWnbULM22/H3r1VmIPtEkXkjcsvkb+tvH65Bg3oR2HTI8AD30TmZePLibogMD0OCxTBaTVaoPvmTYSNvf5ptw4RO5MLqqaNRUene6GyfpDiHC5UsvTKlF5buOI5F2wyrM5MTGiC1bTyKzlXg/rHnYcQrK3C80HWJ3FB35fvpWPf4GJftbAuQLdhyDAu2GB6gDunYFAPbN8XtIzuaz1dUKaw/cAr9LIbI1h9wf1y9skphX95ZpDS3X8jmTRxyIXKhVfTnffgAAAuQSURBVFx9tG0a49FrXpmWhPevTzO/bx1XH/WjwvHohK4uH9xStbwz5/Dm0l12v/EAwPYjhW5dY/Xek3hj6S6c98RCq2mTV7xnGCtfnnUcv249Zn7vTPLU+Xjk+0y8uXQXxr6xEruPO14A5Q1M6EQB4veHRzo89/pVvdGtZSOH54OZo/oxE95aVeNr3fC/tXbH/vlJBm77Yr3b1/gm45C5N597xre/ZTGhE+lsWCe77XfN3ry6D9o2Mfx2YLloqnsr6+TdrVUjvHlNH+8EGELW2DzDGP3ab5rt+loUCDNNrSwtr55Df65Ce4rlJW//gfd/31vHKB3jGDqRjva8ON5qD1WTm4e1x7QFWRjXvQXOPy/RbrVjXEwkVjw0EqOMGzrHx0SZa6IM6dgUbZvE4Ot19jXgbfVrF1+jseFQo7WXK2BdCuHT1dlYsz8ff1nMeDH9N/1+fQ6GWvzA3nK4AFsOF6BeRBj+MbS9x+MVvbb+TEtLUxkZ7tXMICKDbUcK8OC3m/HhjWloEx+D08VlOFlUho6JhmmU32YcwpiuzdEoOgKdHl8IwLBKdth/taf89UmKw57cs3ZL5h25bURHjOyciGs0Vm6Gqv7J8eapjFosZ8GYxvqjI8OQ9fz4Wt1PRNYrpdK0znHIhSiAdG/VGL/ed765lntcTJQ5mQPAVWlJaNIgChHh1f+0TW1NVk8dbX49uW9r/Gnx/tt/D7Zq+/pVva3ej+3WDIM6OC4pHIqcJXOTPblnsdyi3nxEmHdSL4dciILU3DuHIl1j4Ut8TPXm2zcMbmcujhUbHWG3uOnS3q3wwLebAQDf3TYY/dpx8VNNnffEQpTZjKmb9mT1NCZ0oiDVOykOvY1z4rOevwi9nlmMssoqKCj8cPsQREeGWVWMdOSrWwZi65EC9E+2T+bdWjbC9qPuTQ+siUl9WmHuJu0NqgONbTIHDPPUvYFDLkQhIDoyHBf3NlSQDA8T9GsXby4qZmLqNc67e5j5mIhgSKcE3Hp+R6u28+8Zhqcu7obvbx+ML24eiCtS27gsh/DFzQPdjndSn+De+s/dZxY1xYROFCL+e0UvrH38AtSLsF641Cg6AreN6Iivbx0EAOjRujFcjQh0b9UY/xzWHjFRERiWkoDXruqNpQ+MwHUD21q1s3wgOCwlAYvvP9+tWBtEBffggZdGXJjQiUJFZHgYmsVG2x0XEUwd3wVdWlTPbf/234Nx7YCkGieeaZN7ml9Pv9zwOnv6RHNib2ex4vapi7tZfdZyP1jbMeZ7Rgfu9n5aOIZORD6TltwEaRpj5jVxzYC2dsfEuFNoeJjgn8PaIzG2HqqUwnnNY6FU9erOsDDBl7cMxCers3FVWhJSmjWs0UYa/o4JnYgCwvvX93NYR8X2GewlvavHyqssHhT2bhOH8DCxWpRj6a5RnTCuewtcMuMPXDugLWavPQgAmDq+Cw7lFyM+JgozVgTPDwB3MaETkUeN694C47prlw820eqfhoWJ26VoI8PD0LNNY/zxyCi0jquPsd2a4eDJYvPqS6UUJvVphbFvON8hSS//GOL5VaIAx9CJyIdMZQ4Gdqj5cM6XtwzE5amtAQCmdVNt4mMgIhjdpbnVUnoRQUrzWGQ9f1Hdg/aCDone2WOWCZ2IfCY8TLDovvMx63rNletODe2UYF4VG+tgRyhb0ZHhmHvnUKtjqW3jsG/aBKx8eBQynhiDN6+2Lmr2wQ01j62mpnhpf1nWciGigHGuohKfpx/AP4YkW5U3cNepojLUjwq3qzn/j4/X4redefji5oEYlpKA1XtO4LoP12Bst+ZYsv245rWuSmuDbzNyahzDc5O644bByTX+nImzWi5M6EQU8krKKpFzqtjhDkPXzvrLrozCNf2TcGVaEtbsP4mXf91pPp7WLh7lVQrv/S0VmTkFVrXUH5/QFTcNrd0PIxMmdCKiOjhyugRDpi8HAGx7dhxeWbQT941JQVxMFA7lF2P4yysw8+/9MDwlAQ3qWc81qapS6PDYAgCe2X+UCZ2IqI62HylEVISgU7Oa7xP6/foctG0SY1f8rDacJXROWyQickO3VrXf4m9KP+88BLXFWS5EREHCrYQuIheJyE4R2SMiUzXOi4i8ZTyfKSKpng+ViIiccZnQRSQcwDsAxgPoBuBaEelm02w8gBTj160A3vNwnERE5II7PfQBAPYopfYppcoAfA1gkk2bSQA+UwZ/AYgTkZYejpWIiJxwJ6G3BmC5fXiO8VhN20BEbhWRDBHJyMvLq2msRETkhDsJXauOju1cR3faQCk1SymVppRKS0xMdCc+IiJykzsJPQdAksX7NgBsN/tzpw0REXmROwl9HYAUEWkvIlEArgHws02bnwHcYJztMghAgVLqqIdjJSIiJ1wuLFJKVYjIXQAWAQgH8JFSapuI3GY8PxPAAgATAOwBUAzgJlfXXb9+/QkROVDLuBMAnKjlZ/UQSPEyVu8IpFiBwIo31GJt5+iEbkv/60JEMhwtffVHgRQvY/WOQIoVCKx4GWs1rhQlIgoSTOhEREEiUBP6LL0DqKFAipexekcgxQoEVryM1Sggx9CJiMheoPbQiYjIBhM6EVGQCLiE7qqUr49i+EhEckVkq8WxJiKyRER2G/+Mtzj3qDHenSIyzuJ4PxHZYjz3loholVCoa6xJIrJCRHaIyDYRuddf4xWRaBFZKyKbjbE+66+xWtwnXEQ2isi8AIg123ifTSKS4c/xikiciHwvIlnGv7uD/TFWEels/O9p+ioUkft0i1UpFTBfMCxs2gugA4AoAJsBdNMhjvMBpALYanHsZQBTja+nAviv8XU3Y5z1ALQ3xh9uPLcWwGAYauEsBDDeC7G2BJBqfB0LYJcxJr+L13jdhsbXkQDWABjkj7FaxPwAgK8AzPPnvwfG+2QDSLA55pfxAvgUwC3G11EA4vw1VouYwwEcg2Hhjy6xeuUb8+J/sMEAFlm8fxTAozrFkgzrhL4TQEvj65YAdmrFCMOK28HGNlkWx68F8L4P4p4LYKy/xwsgBsAGAAP9NVYYahYtAzAa1QndL2M1Xjsb9gnd7+IF0AjAfhgnbfhzrDbxXQjgTz1jDbQhF7fK9OqkuTLWrzH+2cx43FHMrY2vbY97jYgkA+gLQ8/XL+M1DmFsApALYIlSym9jBfAmgP8AqLI45q+xAoYKqItFZL2I3OrH8XYAkAfgY+Nw1oci0sBPY7V0DYDZxte6xBpoCd2tMr1+xlHMPv1eRKQhgB8A3KeUKnTWVOOYz+JVSlUqpfrA0PsdICI9nDTXLVYRuRhArlJqvbsf0Tjm678HQ5VSqTDsMHaniJzvpK2e8UbAMKT5nlKqL4AiGIYtHNH9v60YChdeCuA7V001jnks1kBL6P5cpve4GHdpMv6ZazzuKOYc42vb4x4nIpEwJPMvlVJz/D1eAFBKnQbwG4CL/DTWoQAuFZFsGHbxGi0iX/hprAAApdQR45+5AH6EYTcyf4w3B0CO8bczAPgehgTvj7GajAewQSl13Phel1gDLaG7U8pXLz8DuNH4+kYYxqpNx68RkXoi0h6GfVfXGn8NOyMig4xPs2+w+IzHGK/9PwA7lFKv+3O8IpIoInHG1/UBjAGQ5Y+xKqUeVUq1UUolw/D3cLlS6u/+GCsAiEgDEYk1vYZhvHerP8arlDoG4JCIdDYeugDAdn+M1cK1qB5uMcXk+1i99YDAiw8eJsAwU2MvgMd1imE2gKMAymH4yXozgKYwPCDbbfyziUX7x43x7oTFk2sAaTD8o9oLYAZsHgJ5KNZhMPzqlglgk/Frgj/GC6AXgI3GWLcCeMp43O9itYl7JKofivplrDCMS282fm0z/dvx43j7AMgw/l34CUC8H8caA+AkgMYWx3SJlUv/iYiCRKANuRARkQNM6EREQYIJnYgoSDChExEFCSZ0IqIgwYRORBQkmNCJiILE/wOdfD5nEFcHUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(log_dict['train_loss_per_batch'], label='Minibatch cost')\n",
    "plt.plot(np.convolve(cost_list, \n",
    "                     np.ones(200,)/200, mode='valid'), \n",
    "         label='Running average')\n",
    "\n",
    "plt.ylabel('Cross Entropy')\n",
    "plt.xlabel('Iteration')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.arange(1, NUM_EPOCHS+1), log_dict['train_acc_per_batch'], label='Training')\n",
    "plt.plot(np.arange(1, NUM_EPOCHS+1), log_dict['valid_acc_per_batch'], label='Validation')\n",
    "\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.set_grad_enabled(False):\n",
    "    \n",
    "    train_acc = compute_accuracy(model=model,\n",
    "                                 data_loader=test_loader,\n",
    "                                 device=DEVICE)\n",
    "    \n",
    "    test_acc = compute_accuracy(model=model,\n",
    "                                data_loader=test_loader,\n",
    "                                device=DEVICE)\n",
    "    \n",
    "    valid_acc = compute_accuracy(model=model,\n",
    "                                 data_loader=valid_loader,\n",
    "                                 device=DEVICE)\n",
    "    \n",
    "\n",
    "print(f'Train ACC: {valid_acc:.2f}%')\n",
    "print(f'Validation ACC: {valid_acc:.2f}%')\n",
    "print(f'Test ACC: {test_acc:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%watermark -iv"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "convnet-vgg16.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "371px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}