3c_ALL-IDB classification using pre-trained networks-Part3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ALL-IDB classification using Inceptionv3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "import struct\n",
    "import torch\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import TensorDataset,DataLoader\n",
    "from torchvision import models\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import copy\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Datapath = 'ALL_IDB2/img/'\n",
    "listing = os.listdir(Datapath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ALL_IDB2 dataset has 260 images in total\n",
    "TrainImages = torch.FloatTensor(200,3,299,299)\n",
    "TrainLabels = torch.LongTensor(200)\n",
    "TestImages = torch.FloatTensor(60,3,299,299)\n",
    "TestLabels = torch.LongTensor(60)\n",
    "\n",
    "# First 200 images are used for training and the remaining 60 for testing\n",
    "img_no = 0\n",
    "for file in listing:\n",
    "    im=Image.open(Datapath + file)\n",
    "    im = im.resize((299,299))\n",
    "    im = np.array(im)   \n",
    "    if img_no < 200:\n",
    "        TrainImages[img_no] = torch.from_numpy(im).transpose(0,2).unsqueeze(0)\n",
    "        TrainLabels[img_no] = int(listing[img_no][6:7])\n",
    "    else:\n",
    "        TestImages[img_no - 200] = torch.from_numpy(im).transpose(0,2).unsqueeze(0)\n",
    "        TestLabels[img_no - 200] = int(listing[img_no][6:7])\n",
    "    img_no = img_no + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([200, 3, 299, 299])\n",
      "torch.Size([200])\n",
      "torch.Size([60, 3, 299, 299])\n",
      "torch.Size([60])\n"
     ]
    }
   ],
   "source": [
    "print(TrainImages.size())\n",
    "print(TrainLabels.size())\n",
    "print(TestImages.size())\n",
    "print(TestLabels.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Creating pytorch dataset\n",
    "trainDataset = TensorDataset(TrainImages, TrainLabels)\n",
    "testDataset = TensorDataset(TestImages, TestLabels)\n",
    "# Creating dataloader\n",
    "BatchSize = 32\n",
    "trainLoader = DataLoader(trainDataset, batch_size=BatchSize, shuffle=True,num_workers=4, pin_memory=True)\n",
    "testLoader = DataLoader(testDataset, batch_size=BatchSize, shuffle=True,num_workers=4, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU is available!\n"
     ]
    }
   ],
   "source": [
    "# Check availability of GPU\n",
    "use_gpu = torch.cuda.is_available()\n",
    "if use_gpu:\n",
    "    print('GPU is available!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Inception3 (\n",
      "  (Conv2d_1a_3x3): BasicConv2d (\n",
      "    (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True)\n",
      "  )\n",
      "  (Conv2d_2a_3x3): BasicConv2d (\n",
      "    (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True)\n",
      "  )\n",
      "  (Conv2d_2b_3x3): BasicConv2d (\n",
      "    (conv): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "  )\n",
      "  (Conv2d_3b_1x1): BasicConv2d (\n",
      "    (conv): Conv2d(64, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True)\n",
      "  )\n",
      "  (Conv2d_4a_3x3): BasicConv2d (\n",
      "    (conv): Conv2d(80, 192, kernel_size=(3, 3), stride=(1, 1), bias=False)\n",
      "    (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "  )\n",
      "  (Mixed_5b): InceptionA (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d (\n",
      "      (conv): Conv2d(192, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d (\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_5c): InceptionA (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d (\n",
      "      (conv): Conv2d(256, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d (\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_5d): InceptionA (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch5x5_1): BasicConv2d (\n",
      "      (conv): Conv2d(288, 48, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(48, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch5x5_2): BasicConv2d (\n",
      "      (conv): Conv2d(48, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6a): InceptionB (\n",
      "    (branch3x3): BasicConv2d (\n",
      "      (conv): Conv2d(288, 384, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(64, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(64, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6b): InceptionC (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d (\n",
      "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d (\n",
      "      (conv): Conv2d(128, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(128, 128, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d (\n",
      "      (conv): Conv2d(128, 128, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d (\n",
      "      (conv): Conv2d(128, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6c): InceptionC (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d (\n",
      "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d (\n",
      "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6d): InceptionC (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d (\n",
      "      (conv): Conv2d(160, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 160, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d (\n",
      "      (conv): Conv2d(160, 160, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(160, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d (\n",
      "      (conv): Conv2d(160, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_6e): InceptionC (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_2): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7_3): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_3): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_4): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7dbl_5): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (AuxLogits): InceptionAux (\n",
      "    (conv0): BasicConv2d (\n",
      "      (conv): Conv2d(768, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(128, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (conv1): BasicConv2d (\n",
      "      (conv): Conv2d(128, 768, kernel_size=(5, 5), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(768, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (fc): Linear (768 -> 2)\n",
      "    (softmax): LogSoftmax ()\n",
      "  )\n",
      "  (Mixed_7a): InceptionD (\n",
      "    (branch3x3_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_2): BasicConv2d (\n",
      "      (conv): Conv2d(192, 320, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7x3_1): BasicConv2d (\n",
      "      (conv): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7x3_2): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(1, 7), stride=(1, 1), padding=(0, 3), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7x3_3): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(7, 1), stride=(1, 1), padding=(3, 0), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch7x7x3_4): BasicConv2d (\n",
      "      (conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_7b): InceptionE (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(1280, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_1): BasicConv2d (\n",
      "      (conv): Conv2d(1280, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_2a): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_2b): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(1280, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3a): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3b): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(1280, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (Mixed_7c): InceptionE (\n",
      "    (branch1x1): BasicConv2d (\n",
      "      (conv): Conv2d(2048, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_1): BasicConv2d (\n",
      "      (conv): Conv2d(2048, 384, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_2a): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3_2b): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_1): BasicConv2d (\n",
      "      (conv): Conv2d(2048, 448, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(448, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_2): BasicConv2d (\n",
      "      (conv): Conv2d(448, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3a): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(1, 3), stride=(1, 1), padding=(0, 1), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch3x3dbl_3b): BasicConv2d (\n",
      "      (conv): Conv2d(384, 384, kernel_size=(3, 1), stride=(1, 1), padding=(1, 0), bias=False)\n",
      "      (bn): BatchNorm2d(384, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "    (branch_pool): BasicConv2d (\n",
      "      (conv): Conv2d(2048, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "      (bn): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True)\n",
      "    )\n",
      "  )\n",
      "  (fc): Linear (2048 -> 2)\n",
      "  (softmax): LogSoftmax ()\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deepkliv/anaconda2/lib/python2.7/site-packages/torchvision-0.1.9-py2.7.egg/torchvision/models/inception.py:65: UserWarning: src is not broadcastable to dst, but they have the same number of elements.  Falling back to deprecated pointwise behavior.\n"
     ]
    }
   ],
   "source": [
    "# Inceptionv3\n",
    "net = models.inception_v3(pretrained=True)\n",
    "num_ftrs1 = net.fc.in_features\n",
    "num_ftrs2 = net.AuxLogits.fc.in_features\n",
    "net.AuxLogits.fc = nn.Linear(num_ftrs2,2)\n",
    "net.fc = nn.Linear(num_ftrs1,2)\n",
    "net.add_module('softmax',nn.LogSoftmax())\n",
    "net.AuxLogits.add_module('softmax',nn.LogSoftmax())\n",
    "print(net)\n",
    "if use_gpu:\n",
    "    net = net.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Apart from the output of the final fully-connected layer, Inceptionv3 also has an additional auxillary unit which is used only during training. Hence, it must be noted that the output of the model is a tuple with the second element of the tuple being the auxillary output."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define loss function and optimizer: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = nn.NLLLoss() # Negative Log-Likelihood\n",
    "optimizer = optim.SGD(net.parameters(), lr=1e-4, momentum=0.9) # Stochastic gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At Iteration: 1 /20  ;  Training Loss: 0.003051 ; Testing Acc: 50.000 ; Time consumed: 0m 4s \n",
      "At Iteration: 2 /20  ;  Training Loss: -0.027777 ; Testing Acc: 50.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 3 /20  ;  Training Loss: -0.075446 ; Testing Acc: 50.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 4 /20  ;  Training Loss: -0.132468 ; Testing Acc: 56.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 5 /20  ;  Training Loss: -0.193522 ; Testing Acc: 63.333 ; Time consumed: 0m 3s \n",
      "At Iteration: 6 /20  ;  Training Loss: -0.255999 ; Testing Acc: 65.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 7 /20  ;  Training Loss: -0.320975 ; Testing Acc: 70.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 8 /20  ;  Training Loss: -0.395304 ; Testing Acc: 66.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 9 /20  ;  Training Loss: -0.466220 ; Testing Acc: 71.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 10 /20  ;  Training Loss: -0.541281 ; Testing Acc: 75.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 11 /20  ;  Training Loss: -0.613636 ; Testing Acc: 78.333 ; Time consumed: 0m 3s \n",
      "At Iteration: 12 /20  ;  Training Loss: -0.703988 ; Testing Acc: 76.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 13 /20  ;  Training Loss: -0.789103 ; Testing Acc: 80.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 14 /20  ;  Training Loss: -0.887157 ; Testing Acc: 80.000 ; Time consumed: 0m 3s \n",
      "At Iteration: 15 /20  ;  Training Loss: -0.966712 ; Testing Acc: 81.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 16 /20  ;  Training Loss: -1.065354 ; Testing Acc: 78.333 ; Time consumed: 0m 3s \n",
      "At Iteration: 17 /20  ;  Training Loss: -1.175342 ; Testing Acc: 76.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 18 /20  ;  Training Loss: -1.278884 ; Testing Acc: 76.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 19 /20  ;  Training Loss: -1.390047 ; Testing Acc: 76.667 ; Time consumed: 0m 3s \n",
      "At Iteration: 20 /20  ;  Training Loss: -1.470201 ; Testing Acc: 75.000 ; Time consumed: 0m 3s \n",
      "Training completed in 1m 7s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEPCAYAAABV6CMBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHnRJREFUeJzt3XuUVOWd7vHvA4gKys0Jl4AgSqIm3oLjZYxnTSkmojGK\njiGYOF6ScXk83s6J44iaaOdkdAWzJo5xYs4QL9E5OhpjRhFFwUtp4tHEBFEIF/EC4gU0KggYL8Dv\n/LE30EJXd1d3V71V1c9nrVq1a/fu3U+XJb9+33fv91VEYGZm1l49UgcwM7P64sJhZmZlceEwM7Oy\nuHCYmVlZXDjMzKwsLhxmZlaW5IVD0nhJCyU9L+miFr7+DUnP5o/fSto7RU4zM8so5X0cknoAzwPj\ngNeBp4FJEbGw2TEHAwsiYpWk8UBTRBycJLCZmSVvcRwILI6IpRHxMXA7cFzzAyLiqYhYlb98Chhe\n5YxmZtZM6sIxHFjW7PWrtF4Y/gGYUdFEZmbWql6pA7SXpMOA04FDU2cxM+vOUheO14CRzV6PyPd9\ngqR9gKnA+Ih4t9TJJHniLTOzMkWEyjk+dVfV08AYSaMk9QYmAdOaHyBpJHAX8PcR8WJbJ4wIP7rg\ncfnllyfP0EgPv59+P2v10RFJWxwRsV7SOcBMsiJ2Q0QskHRm9uWYCnwPGARcJ0nAxxFxYLrUZmbd\nW+quKiLiAWD3Lfb9e7PtM4Azqp3LzMxalrqrympUoVBIHaGh+P3sWn4/00p6A2BXkxSN9PuYmVWa\nJKLMwfHkXVXVsMsuu7B06dLUMWrSqFGjWLJkSeoYZlZHukWLI6+oCRLVPr83Zt1bR1ocHuMwM7Oy\nuHCYmVlZXDjMzKwsLhx17qyzzuKKK65IHcPMuhEPjic2evRobrjhBg4//PAkP7+W3xszqzwPjjeY\n9evXp45gZrYVF46ETjnlFF555RWOOeYY+vXrx49+9CN69OjBjTfeyKhRoxg3bhwAEydOZNiwYQwc\nOJBCocD8+fM3neP000/nsssuA+Cxxx5j55135sc//jFDhgxh+PDh/OIXv0jxq5lZA3PhSOiWW25h\n5MiR3Hfffbz33ntMnDgRgMcff5yFCxfy4IMPAnD00Ufz4osv8uabbzJ27Fi++c1vljzn8uXLWb16\nNa+//jrXX389Z599NqtWrSp5vJlZuVw4AJqaQNr60dTU/uNLHdsOzccYJPH973+f7bffnm233RaA\n0047jT59+rDNNttw2WWX8eyzz7J69eoWz9W7d2++973v0bNnT4466ih22GEHFi1a1OFsZmZbcuGA\n7B/9iK0frRWO9h7bASNGjNi0vWHDBiZPnsyYMWMYMGAAo0ePRhJ//vOfW/zenXbaiR49Nv9n7dOn\nD2vWrOmybGZmLhyJZUuMlN532223ce+99/LII4+wcuVKlixZ0qkFWMzMOsuFI7GhQ4fy0ksvAbRY\nEFavXs22227LwIEDWbt2LRdffHGLxcbMrFqSFw5J4yUtlPS8pItKHPMTSYslzZG0X7UzVtLkyZP5\nwQ9+wKBBg7jrrru2KgqnnHIKI0eOZPjw4ey1114ccsghZZ3fRcbMulrSGwAl9QCeB8YBr5OtQT4p\nIhY2O+Yo4JyI+Iqkg4BrIuLgEueruxsAU/N7Y9a91eMNgAcCiyNiaUR8DNwOHLfFMccBtwBExO+A\n/pKGVDemmZltlLpwDAeWNXv9ar6vtWNea+GYzb785a7KZmZmLWi4FQCbZs3K7qsACkBh552T5jEz\nqyXFYpFisdipc6Qe4zgYaIqI8fnryUBExJRmx/wf4NGIuCN/vRD424hY0cL5Im6/HU46Kbu3YuN+\ncD9+CR7jMOve6nGM42lgjKRRknoDk4BpWxwzDTgFNhWalS0VjU2+/nXYsOGTN+eZmVmXSdpVFRHr\nJZ0DzCQrYjdExAJJZ2ZfjqkRcb+koyW9AKwFTk+Z2cysu+sW63HssssuLF26NEGi2jdq1CiWLFmS\nOoaZJdKRrqpuUTg67ZRT4K67YO3arj+3mVlCLhyVKhw9enxyrGT0aMinCTEzq2f1ODheHzZsgN/9\nbvPrl1/ePJ26mVk348LRXgceuPkqrauuyvb17Jk2k5lZAu6qqoQRI+DVV1OnMDNrk7uqasG//iu8\n9trmrqx77kmdyMysS7nFUQn33AMTJnxy3+c/D/PmpcljZlaCr6qqlcLR3HbbwYcfZtu1ls3Mur2O\nFI6Gm+Sw5nzwQeoEZmZdymMcqUkwxMuLmFn9cOFI6eGHs+c339w8mP7uu2kzmZm1wYUjpXHjsnGP\nCy7YvG/QIN9YaGY1zYPjteTdd7PCAR5IN7Oq8FVV9V44zMyqzDcANrIlS7IurB49YOXK1GnMrBtz\n4agXG9fMiICBA7Mi0sl1g83MOiJZV5WkgcAdwChgCTAxIlZtccwI4BZgCLAB+HlE/KSVczZ+V9WS\nJdm07s01+u9sZhVTV2MckqYAb0fEVZIuAgZGxOQtjhkKDI2IOZJ2AP4IHBcRC0ucs/ELR3O9esH6\n9S4cZtZh9TbGcRxwc759MzBhywMiYnlEzMm31wALgOFVS1jr1q1z0TCzqktZOAZHxArICgQwuLWD\nJe0C7Af8rrXjDJg6NRsD6dkTVq1q+3gzszJUdK4qSbPIxic27QIC+G4Lh5f80znvpvoVcH7e8iip\nqalp03ahUKBQKLQ/cKPYaafsecMGGDAg237uOdh773SZzKwmFItFip28sCblGMcCoBARK/KxjEcj\nYs8WjusFTAdmRMQ1bZyze41xtGXuXNhnn0/u8/tjZs3U2xjHNOC0fPtUoNSKRzcC89sqGtaCvffO\nCsXKldn9H17q1sy6QMoWxyDgl8DOwFKyy3FXShpGdtntMZK+CDwOzCXrygrgkoh4oMQ53eIwMytD\nXV2OWwkuHB1w/PFw993ZWIhn5jXrduqtq8pqwdCh2fPKldmVWL28tpeZtc6Fo7v72c+ycZCrr85e\nr1+fFZDevdPmMrOa5a4q+6QnnoBDD4W+fWFNq1c+m1kD8BiHC4eZWVk8xmGVNWZM1o113XWpk5hZ\nQi4c1n5r12bPZ5+dFZCLLkqbx8yScFeVla9fP1i9evPrb30LbrghXR4z6zB3VVl1vPdediXW4Hxe\nSi8oZdatuMVhZtaNucVhtWPECPjMZ1KnMLMKcOGwynjtNXjhhWwQ/dRTU6cxsy7kwmGVEQFHHplt\n33JLVkBuvTVtJjPrEh7jsMobNgyWL8+2Fy6E3XdPm8fMNvEYh9WmN97IWiAHH+yiYdYA3OIwM+vG\n3OKw+rN6dTb+cdhhqZOYWTslKxySBkqaKWmRpAcl9W/l2B6SZkuaVs2MVgW//W32XCxmBeSKK5LG\nMbO2pWxxTAYeiojdgUeAi1s59nxgflVSWXUddVQ2/rHnntnr7343KyB/+EPaXGZWUsrCcRxwc759\nMzChpYMkjQCOBq6vUi5LYf78rID07Zu9PuCAtHnMrKSU64QOjogVABGxXNLgEsddDVwIlOzKsgay\nZk027tF8EkUzqykVLRySZgFDmu8CAvhuC4dvdTmUpK8AKyJijqRC/v2tampq2rRdKBQoFAplZbYa\nsOOO2cPMulyxWKTYyYlJk12OK2kBUIiIFZKGAo9GxJ5bHHMlcDKwDtge2BH4dUScUuKcvhy3kSn/\nu2HXXeHFF9NmMWsQ9XY57jTgtHz7VOCeLQ+IiEsiYmRE7ApMAh4pVTSsG/jJT7Lnl17Kisghh6TN\nY9ZNpSwcU4AvSVoEjAN+CCBpmKTpCXNZrTr33GwA/dxzs9dPPpkVkKlT0+Yy62Z857jVr2OOgfvu\ny6Y0GTo0dRqzutSRrioXDjOzbqzexjjMKmfoUHjiidQpzBqSWxzWmNTsD6gVKzavj25mn+CuKhcO\na653b/j442x7u+3gL39Jm8esBrmryqy5jz7K7kSX4IMPPtkKMbMOc+Gwxta3L2zYAHff7anbzbqI\nu6rMzLoxd1WZlWvSJOjRA958M3USs7rhwmHd2+zZ2d3oQ4ZAnz6p05jVBRcO696efz67XFfKrrqS\nYPfdU6cyq2kuHGaDB28eQIesmJx4YtpMZjXMg+NmW7r5Zjj11NQpzKrCNwC6cJiZlcVXVZlVkpTd\njf7++6mTmCXlwmHWXhunMOnbFwYOTJ3GLJlkhUPSQEkzJS2S9KCk/iWO6y/pTkkLJP1J0kHVzmoG\nwIcfwtKlWctj5crseY89Uqcyq7qULY7JwEMRsTvwCHBxieOuAe7P1yPfF1hQpXxmWxs5MrsC6/77\ns9eLFqXNY5ZAssFxSQuBv42IFZKGAsWI2GOLY/oBz0TEbu08pwfHzczKUG+D44MjYgVARCwHWlow\nYTTwZ0k3SZotaaqk7aua0qwjHnkkdQKziqlo4ZA0S9JzzR5z8+djWzi8paZCL2As8NOIGAu8T9bF\nZVa79twTxo3LxkBmz06dxqzL9arkySPiS6W+JmmFpCHNuqpammXuVWBZRPwhf/0r4KLWfmZTU9Om\n7UKhQKFQKDe2WecsWADDhsHy5bD//tCrF6xa5bmwrCYUi0WKxWKnzpFyjGMK8E5ETJF0ETAwIrZq\nTUh6DDgjIp6XdDnQJyJaLB4e47Ca06fP5pUHv/ENuPXWtHnMtlBvYxxTgC9JWgSMA34IIGmYpOnN\njjsPuFXSHLKrqq6selKzjnr/fXj11azbavz41GnMuoSnHDEz68Yq1uKQtJukbfPtgqTzJA3oSEgz\nM6tv7e2qugtYL2kMMBXYGbitYqnMuoP778+6sI4+OnUSs7K0t3BsiIh1wPHAtRFxITCscrHMuoHP\nfjZ7njEjW7524yC6WY1rb+H4WNJJwKnAxoHrbSoTyaybGDMmW7Z2v/2y5z594NiWbnEyqy3tLRyn\nA38DXBERL0saDfxH5WKZdSPPPAOLF2fb997rq6+s5pV9VZWkgcDOEfFcZSJ1nK+qsrp3wgnZvR7b\ne2Ydq46KrQAoqQgcS3an+R/J7vJ+IiK+04GcFePCYWZWnkreANg/It4DTgBuiYiDgCPKDWhmHfTw\nw6kTmG3S3sLRS9IwYCKbB8fNrBquuQaOOAJ69swWkDJLrL2F438DDwIvRsTTknYFFlculpltcv75\n8Nd/nS0gNXAgHHJI6kTWzXnKEbN6sXIl7LRTVkAAfvMbOPTQtJms7lVyypERkv5L0pv54y5JIzoW\n08w6ZMAAWL8ezjore3311WnzWLfV3quqZpFNMbLx3o2TgW+2tt5GCm5xmJmVp5KX486JiP3a2pea\nC4eZWXkqeTnu25JOltQzf5wMvF1+RDOrmA8+yCZNPOaY1EmswbW3xTEKuJZs2pEA/h9wbkQsq2y8\n8rjFYd3aypUwaFA275UE8+fDHnukTmU1rmItjohYGhHHRsSnImJwREwA/q5DKXOSBkqaKWmRpAcl\n9S9x3P+SNE/Sc5JuldS7Mz/XrGENGJBdcXXCCVnx2HNP2HXX1KmsAXX4clxJr0TEyA7/4GzN8bcj\n4qpSa45L+jTwW2CPiPhI0h3AfRFxS4lzusVhBlm31YAB8OGH0L+/bxy0kqq95nhZP6gFxwE359s3\nAxNKHNcT6CupF9AHeL2TP9es8W23XVY8fvQjmDkzdRprMClbHO9ExKBSr5vtPw+4AngfmBkRf9/K\nOd3iMDMrQ0daHL3aOOFqssHwrb4EtDnvc37/x5Atvi+A77Zw+FY/J1/X/DhgFLAK+JWkb0SEl601\n64w5c7IFpMw6oNXCERE7dubkrd0gKGmFpCERsULSULKp2rd0BPBSRLyTf8+vgUNoZb3zpqamTduF\nQoFCodCx8GaN6r334AtfyCZNXLsWtt02dSKromKxSLFY7NQ5ks1VlQ+OvxMRU1oZHD8QuAE4APgQ\nuAl4OiJ+WuKc7qoya4+TT84WjAKYMgX+6Z/S5rFkKnbneCVIGgT8EtgZWApMjIiV+fTtP4+IY/Lj\nLgcmAR8DzwD/EBEflzinC4dZe734YrbuOcCIEbCspm7Lsiqpq8JRCS4cZh0wfDi8/jpMnAh33JE6\njVWZC4cLh1nHPPooHHZY6hSWgAuHC4eZWVmqfQOgmTW6t95KncBqkAuHmbXs+ONh8GA48MDUSazG\nuKvKzErbYYfsXo+ePeGdd6Bfv9SJrIu5q8rMutaaNXDqqdmStf37w5VXpk5kNcAtDjNrW/N7Plav\nzloi1hC6fK4qMzMAdtstW+PjwQddNMwtDjOz7sxjHGZWfcuXZ62QF15IncSqxIXDzDrngQeyK68+\n85nN4yDW0Fw4zKxzTjstG/8YOTIbRJfg7LNTp7IK8hiHmXWd+fNh331h3ToYPx5mzEidyNrguapc\nOMxqww9/COec4yuw6oALhwuHmVlZfFWVmdW2vfeGe+5JncI6KVnhkHSipHmS1ksa28px4yUtlPR8\nvsSsmdWr+fNhwgQ48sjUSawTUrY45gLHA4+VOkBSD+DfgCOBzwMnSdqjOvHMrMutXw+77w4zZ0Kf\nPrBqVepE1gHJCkdELIqIxUBrfWsHAosjYmm+zvjtwHFVCWhmlbFwIVx1FfzlLzBgAPzsZ6kTWZlq\nfYxjOLCs2etX831mVs8uvDBbJGq77eCZZ1KnsTJVdJJDSbOAIc13AQFcGhH3VuJnNjU1bdouFAoU\nCoVK/Bgz66y/+qus1WFVVSwWKRaLnTpH8stxJT0KXBARs1v42sFAU0SMz19PBiIippQ4ly/HNTMr\nQz1fjlsq9NPAGEmjJPUGJgHTqhfLzJI4/PDUCawVKS/HnSBpGXAwMF3SjHz/MEnTASJiPXAOMBP4\nE3B7RCxIldnMqmDGDHj0UejRA556KnUaa0Hyrqqu5K4qswbx9NNw0EHZ5Iknngh33pk6UcPylCMu\nHGaNZdQoeOUV2HFHePtt2Gab1IkaTj2PcZiZbW3pUrj0UlizBhYtSp3Gcm5xmJl1Y25xmJlZxblw\nmFl9WrTI3VeJuHCYWX368pdhjz1g6tTUSbodj3GYWX1atw4+/elszquJE+GOO1Inqku+HNeFw6z7\nOeIIePhhGD0aXnopdZq648FxM+t+HnoI/vmf4eWXs64rqzi3OMysMTz5JHzqUzBmTOokdcVdVS4c\nZmZlcVeVmZlVnAuHmTWu2bNh+HD44IPUSRqKC4eZNa7XX88efftmRcS6hAuHmTWuY46BN96AXr1g\n//3hX/4ldaKG4MFxM2t869bBrrvCsmVw1FFw//2pE9WMuhocl3SipHmS1ksaW+KYEZIekfQnSXMl\nnVftnGbWAHr1ytb1mDABfvOb1GnqXrIWh6TdgQ3AvwP/GBFbdUBKGgoMjYg5knYA/ggcFxELS5zT\nLQ4zszJ0pMXRq1Jh2hIRiwAklQwcEcuB5fn2GkkLgOFAi4XDzMwqr24GxyXtAuwH/C5tEjNrOPPm\npU5QVyra4pA0CxjSfBcQwKURcW8Z59kB+BVwfkSsae3YpqamTduFQoFCoVBGYjPrlvbfH/r0gQUL\nYOjQ1GkqqlgsUiwWO3WO5FdVSXoUuKClMY78672A6cCMiLimjXN5jMPMyjdvHhxwAHz0Edx+O3zt\na6kTVU1dXVW1hdZC3wjMb6tomJl12F57wZo18LnPZWt7TJqUOlFNS3k57gRJy4CDgemSZuT7h0ma\nnm9/EfgmcLikZyTNljQ+VWYza2A9e8LcuXDhhdmiUFddlTpRzUreVdWV3FVlZl3ixRdht91Sp6gK\nT6vuwmFmVpZ6HuMwM7M64cJhZtYe116bzbL73HOpkyTnwmFm1h6TJsH228O++8KUKanTJOUxDjOz\ncowfDw8+CF/9KkybljpNp3mMw8ys0h54AKZOzZ4POSR1miRcOMzMynXGGbBoUXazYDfkriozs27M\nXVVmZlZxLhxmZl3pnntgw4bUKSrKXVVmZl3lo49g0CAYNgz++Efo1y91oja5q8rMLKXeveGFF2Dt\nWhgxomEXiHLhMDPrSkOHwiuvwBe+APvtB//5n6kTdTl3VZmZVcp3vpNNVbJiRdaFVYM8O64Lh5nV\nmnfeqdmiAS4cLhxmZmWqq8FxSSdKmidpvaSxbRzbI1/9r/4nhjEzq3MpB8fnAscDj7Xj2POB+ZWN\nY2ZWJd/6Flx3XeoUHZascETEoohYDLTaRJI0AjgauL4qwczMKm233eDcc+EHP8ju/agz9XA57tXA\nhYAHL8ysMVx6KcydC089BfvsA7NmpU5Ull6VPLmkWcCQ5rvICsClEXFvO77/K8CKiJgjqUAbrROA\npqamTduFQoFCoVBeaDOzavjc52D69Oxx5pnw7W9nBaXCisUixWKxU+dIflWVpEeBCyJidgtfuxI4\nGVgHbA/sCPw6Ik4pcS5fVWVm9eeDD+Ddd7OpSqqsrq6q2kKLoSPikogYGRG7ApOAR0oVDTOzurXd\ndkmKRkelvBx3gqRlwMHAdEkz8v3DJE1PlcvMrGa8+iosXpw6xVaSd1V1JXdVmVlDmTYtu3T3zDPh\nkkugb98u/xH13FVlZmZbOvZYePZZWLIE9twT7rwTauCPY7c4zMzqweOPwznnwPDhcN990KNr/u73\nXFUuHGbWyNatg9//Hg45pMtO6cLhwmFmVhaPcZiZWcW5cJiZWVlcOMzMrCwuHGZmVhYXDjMzK4sL\nh5mZlcWFw8zMyuLCYWZmZXHhMDOzsrhwmJlZWVw4zMysLCkXcjpR0jxJ6yWNbeW4/pLulLRA0p8k\nHVTNnGZm9kkpWxxzgeOBx9o47hrg/ojYE9gXWFDpYEanF7O3T/L72bX8fqaVrHBExKKIWEyJ9cYB\nJPUD/ltE3JR/z7qIeK9aGbsz/4/Ztfx+di2/n2nV+hjHaODPkm6SNFvSVEnbpw5lZtadVbRwSJol\n6blmj7n581fbeYpewFjgpxExFngfmFyxwGZm1qbkCzlJehS4ICJmt/C1IcCTEbFr/vpQ4KKIaLHw\nSPIqTmZmZSp3IadelQpSphZDR8QKScskfTYingfGAfNLnaTcX97MzMqX8nLcCZKWAQcD0yXNyPcP\nkzS92aHnAbdKmkN2VdWV1U9rZmYbJe+qMjOz+lLrV1W1i6TxkhZKel7SRanz1DtJSyQ9K+kZSb9P\nnafeSLpB0gpJzzXbN1DSTEmLJD0oqX/KjPWixHt5uaRX8ystZ0sanzJjPZE0QtIj+c3UcyWdl+8v\n6/NZ94VDUg/g34Ajgc8DJ0naI22qurcBKETEFyLiwNRh6tBNZJ/H5iYDD0XE7sAjwMVVT1WfWnov\nAX4cEWPzxwPVDlXH1gHfiYjPA38DnJ3/e1nW57PuCwdwILA4IpZGxMfA7cBxiTPVO9EYn40kIuK3\nwLtb7D4OuDnfvhmYUNVQdarEewmt3DhspUXE8oiYk2+vIZuJYwRlfj4b4R+H4cCyZq9fzfdZxwUw\nS9LTks5IHaZBDI6IFZD9zwsMTpyn3p0jaY6k693t1zGSdgH2A54ChpTz+WyEwmFd74v5DZdHkzVl\nD00dqAH5qpSOuw7YNSL2A5YDP06cp+5I2gH4FXB+3vLY8vPY6uezEQrHa8DIZq9H5PusgyLijfz5\nLeC/yLoDrXNW5De0Imko8GbiPHUrIt6KzZeD/hw4IGWeeiOpF1nR+I+IuCffXdbnsxEKx9PAGEmj\nJPUGJgHTEmeqW5L65H+NIKkv8GVgXtpUdUl8sh9+GnBavn0qcM+W32AlfeK9zP9h2+gE/Pks143A\n/Ii4ptm+sj6fDXEfR3453jVkhfCGiPhh4kh1S9JoslZGkM0scKvfz/JIug0oADsBK4DLgbuBO4Gd\ngaXAxIhYmSpjvSjxXh5G1je/AVgCnLmxf95aJ+mLwONky1pE/rgE+D3wS9r5+WyIwmFmZtXTCF1V\nZmZWRS4cZmZWFhcOMzMriwuHmZmVxYXDzMzK4sJhZmZlceEwa4Ok9fn03c/kz//UheceJWluV53P\nrBpqZelYs1q2Np+7q1J8M5XVFbc4zNrW4hTekl6WNEXSc5KekrRrvn+UpIfz2VtnSRqR7x8s6df5\n/mckHZyfqpekqZLmSXpA0rb58eflC+7Mye+gNqsJLhxmbdt+i66qrzX72rsRsQ/wU7JpbwCuBW7K\nZ2+9LX8N8BOgmO8fC/wp3/8Z4NqI2AtYBfxdvv8iYL/8+P9eqV/OrFyecsSsDZLei4h+Lex/GTgs\nIpbkM46+ERGfkvQWMDQi1uf7X4+IwZLeBIbnC45tPMcoYGa+8hr5+EmviLhS0v3AWrJ5ru6OiLWV\n/23N2uYWh1nnRIntcnzYbHs9m8cev0K2LPJY4Ol8mWSz5PxBNGtba8uUfj1/ngQ8mW8/AZyUb58M\n/Cbffgj4HwCSekja2Iopdf6REfEY2XrQ/YAdyo9u1vV8VZVZ27aTNJvsH/gAHoiIS/KvDZT0LPAB\nm4vFecBNkv4ReAs4Pd//P4Gpkr4NrAPOIlvBbquWSt7F9X/z4iLgmoh4ryK/nVmZPMZh1kH5GMf+\nEfFO6ixm1eSuKrOO819d1i25xWFmZmVxi8PMzMriwmFmZmVx4TAzs7K4cJiZWVlcOMzMrCwuHGZm\nVpb/D6YHU+XOPcKZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f973eb9dc90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEPCAYAAABRHfM8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VOXZ//HPFRYJO6gElYIKVtxYtKJVqVFUcLdqFX2s\nVvwpgrhVrWhtAauPD5VarYiVFpenopS6AU+t4hYVSzUoYFQitghlFZWlrBrJ9ftjTuIYs5xJ5uTM\nTL7v12teztmvjGGu3Oc6932buyMiIlKXvLgDEBGR7KCEISIioShhiIhIKEoYIiISihKGiIiEooQh\nIiKhRJ4wzGyImZWa2WIzu7Ga7e3NbKaZLTCzEjP7SdK2pWa20Mzmm9lbUccqIiI1syj7YZhZHrAY\nGASsAoqBoe5emrTPTUB7d7/JzHYBPgQK3P0rM1sCHOLu6yMLUkREQom6hTEA+Mjdl7l7GTANOL3K\nPg60C963Az5396+CZWuEGEVEJISov4z3AJYnLa8I1iWbCOxvZquAhcDVSdsceMHMis3s0kgjFRGR\nWjWPOwBgMDDf3Y81s54kEkQfd98MHOnuq81s12D9InefE2+4IiJNU9QJYyXQPWm5W7Au2cXAHQDu\n/i8z+xjoDcxz99XB+k/N7GkSt7i+lTDMTANiiYikyN0tlf2jviVVDPQysx5m1hIYCsysss8y4DgA\nMysAvgssMbPWZtY2WN8GOAF4r6YLubteaXiNGTMm9hhy6aXPU59npr7qI9IWhrvvMLNRwGwSyWmK\nuy8ys+GJzT4ZuA142MzeDQ77mbuvM7O9gKeD1kNzYKq7z44yXhERqVnkNQx3fw7Yt8q6B5LeryZR\nx6h63MdAv6jjExGRcPTIqnxDYWFh3CHkFH2e6aXPM16RdtxrLGbmufBziIg0FjPDUyx6Z8JjtZHZ\nc889WbZsWdxhxK5Hjx4sXbo07jBEJMvldAsjyKAxRJRZ9DmISFX1aWGohiEiIqEoYYiISChKGCIi\nEooShoiIhKKEEZO99tqLl19+uUHneOSRRxg4cGCaIhIRqZ0SRhZzd8xSeshBRKTelDBicOGFF/Lv\nf/+bU089lfbt2zNhwgTefPNNjjzySDp16kT//v159dVXK/d/+OGH6dmzJ+3bt6dnz548/vjjlJaW\nMmLECObOnUu7du3o3LlzjD+RiDQF6ocRk7322osHH3yQY445hlWrVtGnTx+mTp3K4MGDeemllzj3\n3HP58MMPyc/PZ7fdduPtt9+mV69efPLJJ6xbt4799tuPRx55hClTpvDaa6/Veq1M/hxEJB7q6Z0i\nG5ee2zk+pn5fxhVf4o8++ignn3wygwcnxmAcNGgQ3/ve93j22Wc566yzaNasGSUlJXTr1o2CggIK\nCgrSEreISCqadMKo7xd9ui1btozp06cza9YsIJFIvvrqK4499lhat27Nn//8Z+68806GDRvGUUcd\nxYQJE9h3333rOKtI5hg+azgHdjmQKw+7Mu5QpAGadMKIU3Kx+jvf+Q4XXnghDzzwQLX7Hn/88Rx/\n/PF88cUX/PznP+eyyy7j1VdfVcFbMt79xfcz8tmRlcu9OvfixH1OjDEiaQgVvWPStWtXlixZAsAF\nF1zArFmzmD17NuXl5Wzfvp1XX32VVatWsXbtWmbOnMnWrVtp0aIFbdu2JS8v8b+toKCAFStWUFZW\nFuePIvIt81bOo+WvWjLy2ZEYxuSTJ9PMmnHSYyfx4Wcfxh2e1JOK3jGZOXMmV155JZs2beKWW25h\n4MCB3HDDDZSUlNC8eXMGDBjA/fffT/PmzRk6dCgLFy7EzOjXrx+TJk2id+/elJWVceaZZ/L3v/+d\nZs2asXbt2mqvlcmfg+SWzV9upvtvu7N++3oAhvUdxpQzpgCwcetGOt3ZCcfZcMMGOrTuEGeoTV59\nit5KGE2APgdpDH0m9aHk0xIA+hb0ZcHlC761z4effUjv+3qTZ3lsv2k7LVq0aOwwJaCE8e31+qJE\nn4NE68KnLuRPJX8CYOf8nVl6zVLatmxb4/4v/OsFTnj0BPKb57P151sbK0ypIiOHNzezIWZWamaL\nzezGara3N7OZZrbAzErM7CdhjxWR+NxffD82zvhTyZ9okdeChcMX8tnPPqs1WQAc3/N4Jg6ZyLav\ntlFwpx4RzyaRtjDMLA9YDAwCVgHFwFB3L03a5yagvbvfZGa7AB8CBUB5XccmnUMtjFroc5B0mrdy\nHkc8eARl5WUYxgMnP8Cl37s05fNc/ber+d1bv2P/Xffn/ZHvRxCp1CYTWxgDgI/cfZm7lwHTgNOr\n7ONAu+B9O+Bzd/8q5LEi0kg2f7mZzuM7c+gfD6WsvIxhfYdRPqa8XskC4J4T7+GkXifxwacfMPhP\ng9McrUQh6oSxB7A8aXlFsC7ZRGB/M1sFLASuTuFYEanDli+3NPgcfSb1od0d7Vi/fT19C/riY7zy\n6aeG+Ot//ZUDuxzI7CWzGfXsqAafT6KVCR33BgPz3f1YM+sJvGBmfeIOSiQXdL2zK59s/SQt5+qc\n35ll1yyrs0aRqpIRJXSd0JX7iu9jv53344rDrkjr+SV9ok4YK4HuScvdgnXJLgbuAHD3f5nZx0Dv\nkMdWGjt2bOX7wsJCCgsL6dGjh3pDAz169Ig7BIlB//v788nWTyhoXcCaG9bEHU6t1ly/hja3t2HU\nc6PYu/Pe6g0egaKiIoqKihp0jqiL3s1IFLEHAauBt4Dz3H1R0j73AWvdfZyZFQDzgL7AxrqOTTpH\ntUVvkabqR9N/xBOLnqBVs1Zsu2Vb3OGEUlZWRv4d+ezwHZReUcq+u2i8tChlXNHb3XcAo4DZwPvA\nNHdfZGbDzeyyYLfbgCPM7F3gBeBn7r6upmOjjFckF9xadCtPLHqCPMvLmmQB0KJFCzaO3ohh9L6v\nNxu3bow7JKkipzvuiTQ1MxbN4IzpZwCZMxpzqv752T/Z57591Bs8YhnXwhCRxlO6prQyWWy7MXta\nFlX12qUXsy+YTbmX0+HXGm8qkyhhiOSA7du3s98D+wGwaPgiWrVqFXNEDXN8z+O578T71Bs8wyhh\niOSA/PH5ADxzzjP07to75mjSY+SAkVw14CrWbl3LAZMOiDscQQlDJOs1u7UZAOOOHsfp++XWYAjq\nDZ5ZVPQWiUjeuDwc56lznuKH+/0wkmu0vr01277axtn7nc1fzvlLJNfIBH3u70PJ2hL6FvRl3v+b\nR/PmmdDnOLtpeHORDNFsXDPKKf96mWas/dlaOud3Tts1Knpx9+vSj/kj5qftvJnqiClHMHfFXAAu\n7ncxD57+YMwRZTclDJEM0Pa/27KlbAt7tt+Tj6/9mN0m7MaaLYme1h126sCG0RsafI3+9/dnwdoF\nWdGLO502b99Mv8n9+Nf6f9HMmnHPkHu4YoCGEqkPJQyRmHX/TXeWb15O2xZt2XTzpsr127Zto+OE\njnxZ/iUA/Qr6Mf/y+rUKsrEXd7q9u+Zdjn74aDZ8sYE2Ldrw3AXPcVT3o+IOK6uoH4ZIjH7w4A9Y\nvnk5za35N5IFQH5+Pl/84gtev+h1ABZ8sgAbZ1zxf6n9dZytvbjTrU/XPqwfvZ6pP5zKlzu+ZOBD\nA9njrj34bPNncYeW09TCEEmDEbNG8Pt3fg+E62F9x2t3cPMrN1cuhymM50Iv7qhc9/x1/PYfv8Vx\nBuw+gDcufkOF8TrolpRIDP4w7w9c9tfE0GipfpEf97/H8dLHLwG1F8ZL15RWdszbduO2rO+YF5Xj\n//d4Xvz4RQAu6X8JfzztjzFHlLmUMEQa2TvL3+GQBw8BYMsNW2jdunW9zlNbYXz79u2VHfMWDV+U\nMx3zorJ5+2b6T+7PP9f/U4XxWihhiDSirVu30ubONgAsumIRvXdp2Bd5TYVxG5f4N/3MOc/kXMe8\nKKkwXjslDJFGVPFF/viZjzP0oKFpO++cpXMY+MjAb6wbd/Q4fln4y7Rdoyl5vORxLnrmIsrKy9g5\nf2d6du7ZoPO1a9mOrm27Nugc5x94Pid996QGnaOhlDBEGklFsrjusOuYMGRCJNeoKIyf0vMUZl0w\nK5JrNCU3zL6BicUT2anZTg06T6vmrejUqlO9jy+nnE82f8IFfS5gzNFj2LXNrg2Kp76UMEQaQUUv\n7kF7DuLFi16MOxzJQp9v/ZxbX72VqSVTuf6I67n6sKvJb5HfqDEoYYhErGovbpGG+Ojzjxj90mjm\nrZrH7cfezvkHnU+eNU73OCUMkQhV9OJu17Id/7npP3GHIzlkzr/ncN3s69hRvoMJJ0ygcM/CyK+p\nhCESkR88+ANeX/46za05Zb8sizscyUHuzvT3pzP6pdH0KejD+OPGN/jJu9pk5NAgZjbEzErNbLGZ\n3VjN9uvNbL6ZvWNmJWb2lZl1DLYtNbOFwfa3oo5VpDojZo3g9eWJIT2ULCQqZsa5B55L6RWlDOw+\nkIEPDeSKv17Bp1s+jTu0SpG2MMwsD1gMDAJWAcXAUHcvrWH/U4Br3P24YHkJcIi7r6/jOmphSCQa\n0otbpCE+3/o5v3rtVzz67qORFMYzsYUxAPjI3Ze5exkwDait59F5wONJy4YGSJSYvLP8ncpkseWG\nLTFHI03Nzq135u4hdzP3krnMWzWP3vf15tF3H6Xcy+s+OCJRfxnvASxPWl4RrPsWM8sHhgBPJq12\n4AUzKzazSyOLUqSKLVu2VA75seiKRfUe8kOkofbZeR+eOOcJHjvzMSa+NZEBfxhA0dKiWGLJpOEc\nTwXmuHvy7DJHuvtqM9uVROJY5O5zqjt47Nixle8LCwspLCyMMlbJcW0ntAUSvbijLDyKhHVk9yOZ\ne8lcpr8/nUtnXcpLF75E9w7dQx9fVFREUVFRg2KIuoZxODDW3YcEy6MBd/fx1ez7FDDd3afVcK4x\nwCZ3v6uabaphSNpU9OIecfAIJp06KeZoRL6t3Msb3F8jE2sYxUAvM+thZi2BocDMqjuZWQfgaGBG\n0rrWZtY2eN8GOAF4L+J4pYnLG5f4J3FEtyOULCRjNVbnvqoivSXl7jvMbBQwm0RymuLui8xseGKz\nTw52PQN43t2TpxArAJ42Mw/inOrus6OMV5q2/NvycZxubbvxxiVvxB2OSMZRxz0RYLc7d2PN1jXk\nN89n68+3xh2OSOQy8ZaUSMYbMHkAa7auIY88JQuRWihhSJP2k6d+QvHqYgB2jNkRczQimU0JQ5qs\ne/9xL4+UPAKoF7dIGKphSJP05rI3OfzhwwHYfP1m2rRpE3NEIo1LNQyRELZs2VKZLEpGlChZiISk\nhCFNTkUv7odPe5gDuxwYczQi2UMJQ5qUil7cIw8eyUX9L4o5GpHsohqGNBl54/JwnCO6HaGOedLk\nacY9kRrk35bP9h3b6da2G8uvW173ASI5TkVvkWrsdudubN+xnfzm+UoWIg1QZ8Iwsz+b2WAzSykT\niWQC9eIWSZ8wLYyHgGHAYjO7zcx6RRyTSFqoF7dIetWZMNz9OXc/l8R0q2uAV8zsNTP7sZll0gRM\nIpXUi1sk/UIVvc2sE3A+cCHwGfAYcBSwj7sfF2mEIajoLcnWbF7Dbr/ZDVAvbpGa1KfoXWcLwcz+\nAhwETAXOcvcVwaapZjY/9TBFolWRLBZevlDJQiSNwtxSmgy8WN2f8O7eP/0hidTf8BnDAei8U2f6\nFPSJORqR3BKm6N0T6FCxYGadzOyy6EISqb/JCxKTOH4++vOYIxHJPWESxuXuvqFiwd3XAyOiC0mk\nfjre0RGAaw+7NuZIRHJTmITRLHnBzPKAFtGEI1I/azavYeOXGwG4a8hdMUcjkpvCJIwXzOxxMzva\nzI4mUfx+MewFzGyImZWa2WIzu7Ga7deb2Xwze8fMSszsKzPrGOZYkQoVhe7V162OORKR3FXnY7Vm\n1gwYCQwKVr0APODuX9V58kRrZHFw7CqgGBjq7qU17H8KcI27H5fKsXqstmkbPmM4kxdMpvNOnVW7\nEAkpksdq3X0HcG/wStUA4CN3XxYEOA04Hag2YQDnAY/X81hpolToFmkcYcaS6mlm08zs3eDW0GIz\nWxzy/HsAyaO9rQjWVXedfGAI8GSqx0rmuL3odm5+4eZGu54K3SKNJ0w/jIeB24AJwInAxUAU939O\nBeYkP5GVirFjx1a+LywspLCwMD1RSUpuefUWALaVbeO3J/020mup0C0SXlFREUVFRQ06R5gaxtvu\nfoiZlbj7QcG6ee7+vTpPbnY4MNbdhwTLowF39/HV7PsUMN3dp9XjWNUwMkDFbHYVnjjrCc468KzI\nr7f6utV0bds1suuI5KKo5sP4IihA/8vMLjezU4F2Ic9fDPQysx5m1hIYCsysupOZdQCOBmakeqxk\nhofeeQgAwyi9NFFmOvvJs1m1aVUk10vu0a1kIdI4wrQwDgM+ADoBtwPtgV+7e6g5Ls1sCHAPieQ0\nxd3/x8yGk2gtTA72uQgY7O7n13VsDddQCyNmFX/tV4wMO3XhVC545gIA/vPT/9CuXdi/Mep3PRFJ\nTdqnaA0eqb3d3Uc3NLgoKWHEq+fdPVmycQn7dN6HxVd+/TzEyP8byf1v3w+k94u94x0d2fjlRq49\n7FrVLkTqKZI5vc3sTXc/rEGRRUwJI161/bU/8I8DmbNyTo3bU5U8dLlaFyL1F1UN420ze8rMzjOz\n0ype9YxRckxFsnj8jMer3f76/3ud3dvsDkDeuIZPIa8e3SLxCfMvuB2wBTgJ+FHwOjvKoCQ7JBe6\nh/YdWuN+K69fyU7NdsJxWt/Wut7XU6FbJF6hZtzLdLolFY9UC8954/JwnG7turH8p8vrPqCB1xOR\nmkU1497k6ta7u+bEaMJ63t0TgH067xP6mPIx5dg4Y8WmFRwz5RheueSV0MeqR7dI/MLcknop6fUG\n0AX4IsqgJPMt2bgE4BtPRYVR0TooWlHEtc+G+/JXj26RzJDyLamgE98cdz8impBSp1tSjSu50F1b\n7aImmzZtov1d7YFwvcHVo1sk/aJ6SqqqvYCCehwnOSBsobs27dq1C90bXIVukcwRph/Ger4ebDAP\nWAeMdvfpEccWmloYjSedhecwvcFV6BaJRlQtjF2AXYNXJ3ffO5OShTSeikL3fjvvl5bz/Vff/2LE\nIYnp4StuUSVToVsks4RJGCcDbd19h7u7mXUMZsaTJqai0P3BqA/Sds5Jp0zi8N0PB7452q0K3SKZ\nJ0zCuNXdN1YsBPNV/Cq6kCQT1dWjuyHmXjr3W73B1aNbJPOESRjV3eMKM/GS5Ih0FLrrktwbvNm4\nZoAK3SKZJkzR+2FgLXBfsGoUUODuF0YbWngqekerMQvPFb3BG+t6Ik1VVEXvUcF+M4BnSDwxNTL1\n8CQbpbvQXZfyMeW0zGvJmIFjGuV6IhKexpKSWumxVpHcFEkLw8yeM7OOScudzOyv9QlQskuUhW4R\nyT5hbkkVBE9GAeDu64HdowtJMkFjFLpFJLuESRjlZtatYsHMukcYj2SIYbOGAYmagogIhHs89pfA\nG2b2MolHbAuBEWEvYGZDgLtJJKcp7j6+mn0Kgd8CLYBP3f2YYP1SYCNQDpS5+4Cw15X6a+xCt4hk\nh1BFbzMrAL4fLP7d3deGOnliZNvFwCBgFVAMDHX30qR9OgB/B05w95Vmtou7fxZsWwIcEtwGq+06\nKnqnkQrdIrkvytFqtwP/JtEfo5eZhR3afADwkbsvc/cyYBpwepV9zgeedPeVABXJImApxChpoEK3\niNQkzFNSw0i0AF4Gxgf//e+Q598DSJ6Lc0WwLtl3gc5m9oqZFZvZj5O2OfBCsP7SkNeUerrm2WsA\nFbpFpHphahjXAt8D5rr7QDM7ALg1zTEcDBwLtAHmmtlcd/8ncKS7rzazXUkkjkXuPqe6k4wdO7by\nfWFhIYWFhWkMMbeV/LuEPg/1qVxWoVsk9xQVFVFUVNSgc4QZGqTY3Q81swXAAHf/0szec/cD6zy5\n2eHAWHcfEiyPBjy58G1mNwKt3H1csPxH4G/u/mSVc40BNrn7t4YuVQ2j/pJHiG2Z15IvfqHZd0Wa\ngvrUMMK0MFYHHfdmAc+b2ToSt5bCKCZR8+gBrAaGAudV2WcGcK+ZNQN2Ag4D7jKz1kCeu282szbA\nCcC4kNeVOrS5vQ1bv9paubzhmg106NAhxohEJNPVmTDc/bTg7S/MbBDQAQjV09vdd5jZKGA2Xz9W\nu8jMhic2+2R3LzWz54F3gR3AZHf/wMz2Ap42Mw/inOrus1P+CeUb+k3qx8JPF1YuPzf0OQbvOzjG\niEQkW2gsqSZi9OzRjJ/7dReYYX2GMeWHU2KMSETiFNUtKcliVQva3dt1Z9lPl8UYkYhkKyWMHKaC\ntoikkxJGDlJBW0SiUGfCMLP1QNUCwUZgHnCDuy+NIC6ph4MmHcR7n75XuayCtoikU5gWxn0kHol9\nLFg+D9gTWAg8BBwTSWSSkjMfP7MyWaigLSJRCNNxb6G7962yboG796tuWxz0lJQGDBSR1EQ1+OA2\nMzsz6SJnAhXVU40hkQGmvTsNgFZ5rWKORERyWZgWRi/gXhI9sB14C7iaRG/vQ9391aiDrEtTb2Go\ndSEiqYqkH0YwCOCJNWyOPVk0dRs2bKh7JxGRNAjzlNQuwDAShe7K/d39sujCkrA63dMJgPVX1zrH\nlIhIg4V5SmoG8A9gDomxniQDdezYMe4QRCTHhUkYbdz9usgjkZS1vq01AFNO0iO0IhK9ME9J/c3M\nTog8EknZth3bABh26LCYIxGRpiBMwrgceM7MNpvZOjNbH8yJITHqOzHR/eWEPZXLRaRxhHmstll1\n6909Y+oZTfGxWj1KKyINkdbHas1sH3f/CDighl3eTeVCkj43z74ZgJ1b7RxzJCLSlNTYwjCzKe5+\niZm9Xs1md/cfRBtaeE2thaHWhYg0VFpbGO5+SfD2WHcvq3KhFvWIT9JgwdIFAOSFKj+JiKRPmG+d\nN0Ouk0bQ/5H+AHx+9ecxRyIiTU2NCcPMuphZXyDfzA4ysz7B6yigddgLmNkQMys1s8VmdmMN+xSa\n2Xwze8/MXknl2KZKHfVEpLHV1nHvZBJDgnQjMSdGxb2uTcAvwpzczPKAicAgYBVQbGYz3L00aZ8O\nwflPcPeVwVAkoY5tapqNSzyw9spFr9Sxp4hI+tVWw3gIeMjMznH36fU8/wDgI3dfBmBm04DTgeQv\n/fOBJ919ZXDdz1I4tkkpD0aTL9yzMN5ARKRJClPD6GJm7QHM7Pdm9paZDQp5/j2A5UnLK4J1yb4L\ndDazV8ys2Mx+nMKxTcbuE3YHYET/ETFHIiJNVZixpC5z94nB8CC7AZcCDwKHpDGGg4FjgTbAXDOb\nm+pJxo4dW/m+sLCQwsLCNIWXGVZvWQ3ApNMmxRyJiGSjoqIiioqKGnSOMAmj4mH/k4D/dfeFQX0h\njJVA96TlbsG6ZCuAz9x9O7DdzF4D+oY8tlJywsg1Zz6emPBw3077xhyJiGSrqn9Ijxs3LuVzhPni\nX2hmzwKnkBiIsC1fJ5G6FAO9zKyHmbUEhgIzq+wzAzjKzJqZWWsSM/stCnlsk/D04qcBKL2qyZZv\nRCQDhGlhXEzi9tM/3X1r8BTTJXUcAyTGmzKzUcBsEslpirsvMrPhic0+2d1Lzex5EkON7AAmu/sH\nANUdm+oPmO00X7eIZIo6Bx8EMLOhQE93v93MvgN0cfe3I48upFweGkTDgIhIFOozNEidt6TMbCJw\nDHBBsGoL8PvUw5NUab5uEckkYW5JHeHuB5vZfAB3XxfUFCRimq9bRDJJmKJ3WfBUlAOY2c4Q9CCT\nRqFhQEQkE9Q2llRF6+M+4ElgVzMbB8wBxjdCbE2a5usWkUxT23wY77j7wcH7A4DjSIwn9aK7v9d4\nIdYtF4veKnaLSJTSOh8GXw82iLu/D7xf38AkNf3vTwxhrvm6RSST1NbCWAHcVdOB7l7jtsaWay0M\ntS5EJGrpbmE0A9qS1NKQ6Gm+bhHJVKFqGJkul1oYal2ISGNId8c9tSwamebrFpFMVlsLo7O7r2vk\neOolV1oYFa2L9VevV98LEYlUWlsY2ZIscpGShYhkIt37yBDNxyWeP9B83SKSqZQwMsQOdgCar1tE\nMpcSRgbo8usuAFzSJ9Q0IyIisQg1H0amy/aitx6lFZHGFsl8GBKtUx49BYC9O+wdcyQiIrVTCyNm\nal2ISBzUwsgyjy54FICWmo9KRLJA5C0MMxsC3E0iOU1x9/FVth8NzACWBKuecvfbgm1LgY0kJmwq\nc/cBNVwjK1sYal2ISFzSPfhggwUz9U0EBgGrgGIzm+HupVV2fc3dT6vmFOVAobvn3Byl69fn3I8k\nIjku6ltSA4CP3H2Zu5cB04DTq9mvpixn5Ohts86/6wzAuqvUoV5EskPUX8Z7AMuTllcE66r6vpkt\nMLO/mtn+SesdeMHMis3s0igDjUunTp3iDkFEJJRIb0mF9DbQ3d23mtmJwDPAd4NtR7r7ajPblUTi\nWOTuc6o7ydixYyvfFxYWUlhYGG3UDdDqV60A+P3g38cciYg0FUVFRRQVFTXoHJEWvc3scGCsuw8J\nlkcDXrXwXeWYj4FDqg5+aGZjgE3VzfSXbUVvFbtFJG6Z+FhtMdDLzHqYWUtgKDAzeQczK0h6P4BE\nEltnZq3NrG2wvg1wAvBexPFG7sCJBwJQ+J3CeAMREUlRpLek3H2HmY0CZvP1Y7WLzGx4YrNPBs42\nsxFAGbANODc4vAB42sw8iHOqu8+OMt7G8P7n7wPwyjCNSisi2UU9vRvR9X+7nt+89Rs6tuzI+pv0\nWK2IxKc+t6SUMBqRahcikikysYYhgbeXvA2Aaap0EclSamE0korWxbqr1qnvhYjETi2MLKBkISLZ\nSgmjEeSNS3zML/74xZgjERGpPyWMRuAkbpcN2ntQzJGIiNSfEkbEdv31roDm6xaR7Keid8T0KK2I\nZCIVvTOM5usWkVyiFkaE1LoQkUylFkYG0XzdIpJr1MKIiFoXIpLJ1MLIEJqvW0RykRJGBDRft4jk\nIiWMCGkYEBHJJUoYaab5ukUkV6nonWYqdotINlDRO2b7/25/QPN1i0huUgsjnXGodSEiWSIjWxhm\nNsTMSs0525GXAAAHLklEQVRssZndWM32o81sg5m9E7xuCXtsJrn+b9cD0LFlx5gjERGJRqQtDDPL\nAxYDg4BVQDEw1N1Lk/Y5GrjO3U9L9dikfWNvYah1ISLZJBNbGAOAj9x9mbuXAdOA06vZr7qgwx4b\nO83XLSJNQfOIz78HsDxpeQWJRFDV981sAbASuMHdP0jhWODrv/Dj9PlVn8cdgohIZKJOGGG8DXR3\n961mdiLwDPDdlM/yStL7PYG90hJbStRRT0QyVVFREUVFRQ06R9Q1jMOBse4+JFgeDbi7j6/lmI+B\nQ0gkjVDHZkINQ0Qkm2RiDaMY6GVmPcysJTAUmJm8g5kVJL0fQCKJrQtzrIiINJ5Ib0m5+w4zGwXM\nJpGcprj7IjMbntjsk4GzzWwEUAZsA86t7dgo4xURkZqp456ISBOUibekREQkRyhhiIhIKEoYIiIS\nihKGiIiEooQhIiKhKGGIiEgoShgiIhKKEoaIiISihCEiIqEoYYiISChKGCIiEooShoiIhKKEISIi\noShhiIhIKEoYIiISihKGiIiEooQhIiKhKGGIiEgokScMMxtiZqVmttjMbqxlv0PNrMzMzkxat9TM\nFprZfDN7K+pYRUSkZpEmDDPLAyYCg4EDgPPMrHcN+/0P8HyVTeVAobv3d/cBUcYqCUVFRXGHkFP0\neaaXPs94Rd3CGAB85O7L3L0MmAacXs1+VwJPAGurrDd026xR6R9keunzTC99nvGK+st4D2B50vKK\nYF0lM9sdOMPd7yeRIJI58IKZFZvZpZFGKiIitWoedwDA3UBybSM5aRzp7qvNbFcSiWORu89p3PBE\nRATA3D26k5sdDox19yHB8mjA3X180j5LKt4CuwBbgMvcfWaVc40BNrn7XdVcJ7ofQkQkR7l71bs6\ntYq6hVEM9DKzHsBqYChwXvIO7r53xXszewiY5e4zzaw1kOfum82sDXACMK66i6T6Q4uISOoiTRju\nvsPMRgGzSdRLprj7IjMbntjsk6sekvS+AHg6aD00B6a6++wo4xURkZpFektKRERyR1Y/shq2U6CE\no46SDWNmU8zsEzN7N2ldJzObbWYfmtnzZtYhzhizRQ2f5RgzW2Fm7wSvIXHGmE3MrJuZvWxm75tZ\niZldFaxP6fczaxNG2E6BkhJ1lGyYh0j8PiYbDbzo7vsCLwM3NXpU2am6zxLgLnc/OHg919hBZbGv\ngJ+6+wHA94Ergu/LlH4/szZhEL5ToISnjpINEDzyvb7K6tOBR4L3jwBnNGpQWaqGzxK+3VdLQnD3\nNe6+IHi/GVgEdCPF389s/nKos1OgpEwdJdOvi7t/Aol/tECXmOPJdqPMbIGZ/VG39+rHzPYE+gH/\nAApS+f3M5oQh6Xekux8MnESiyXpU3AHlID1lUn+TgL3dvR+wBvhWnyypnZm1JTEM09VBS6Pq72Ot\nv5/ZnDBWAt2TlrsF66Se3H118N9PgadJ3PaThvnEzAoAzKwr3x4vTUJy90/968c6/wAcGmc82cbM\nmpNIFn9y9xnB6pR+P7M5YVR2CjSzliQ6Bc6s4xipgZm1Dv76IKmj5HvxRpWVjG/eZ58J/CR4fxEw\no+oBUqNvfJbBF1qFM9HvZ6oeBD5w93uS1qX0+5nV/TCCx+ru4etOgf8Tc0hZy8z2ItGqSO4oqc8z\nBWb2GFAI7Ax8AowBngH+AnwHWAac4+4b4ooxW9TwWR5D4t57ObAUGF5x/11qZ2ZHAq8BJST+jTtw\nM/AWMJ2Qv59ZnTBERKTxZPMtKRERaURKGCIiEooShoiIhKKEISIioShhiIhIKEoYIiISihKGSA3M\nbEcwjPb84L8/S+O5e5hZSbrOJ9IYop6iVSSbbQnG1oqKOkFJVlELQ6Rm1Q6lbWYfm9l4M3vXzP5h\nZnsH63uY2UvBaKovmFm3YH0XM3sqWD/fzA4PTtXczCab2Xtm9pyZ7RTsf1Uw0c2CoMezSEZQwhCp\nWX6VW1I/Stq23t37APeRGJ4G4F7goWA01ceCZYDfAUXB+oOB94P1+wD3uvuBwEbgrGD9jUC/YP/L\no/rhRFKloUFEamBm/3H39tWs/xg4xt2XBiOArnb3Xc3sU6Cru+8I1q9y9y5mthbYI5joq+IcPYDZ\nwUxnBPWR5u7+32b2LLCFxDhUz7j7luh/WpG6qYUhUj9ew/tUfJH0fgdf1xRPJjH98MFAcTAdsUjs\n9IsoUrPapgM9N/jvUGBu8P4N4Lzg/QXA68H7F4GRkJiL3swqWi01nb+7u79KYr7l9kDb1EMXST89\nJSVSs1Zm9g6JL3YHnnP3m4NtncxsIbCdr5PEVcBDZnY98ClwcbD+GmCymV0CfAWMIDFj3LdaJsGt\nrEeDpGLAPe7+n0h+OpEUqYYhkqKghnGIu6+LOxaRxqRbUiKp019Z0iSphSEiIqGohSEiIqEoYYiI\nSChKGCIiEooShoiIhKKEISIioShhiIhIKP8fKtyh7oj1vXUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f97d0205f90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "iterations = 20\n",
    "trainLoss = []\n",
    "testAcc = []\n",
    "start = time.time()\n",
    "for epoch in range(iterations):\n",
    "    epochStart = time.time()\n",
    "    runningLoss = 0    \n",
    "    net.train(True) # For training\n",
    "    for data in trainLoader:\n",
    "        inputs,labels = data\n",
    "        # Wrap them in Variable\n",
    "        if use_gpu:\n",
    "            inputs, labels = Variable(inputs.float().cuda()), \\\n",
    "                Variable(labels.long().cuda())\n",
    "        else:\n",
    "            inputs, labels = Variable(inputs), Variable(labelslong()) \n",
    "        inputs = inputs/255.0\n",
    "        # Initialize gradients to zero\n",
    "        optimizer.zero_grad()\n",
    "        # Feed-forward input data through the network\n",
    "        outputs = net(inputs)\n",
    "        # Compute loss/error\n",
    "        loss1 = criterion(outputs[0], labels)\n",
    "        loss2 = criterion(outputs[1], labels)\n",
    "        loss = loss1+loss2\n",
    "        # Backpropagate loss and compute gradients\n",
    "        loss.backward()\n",
    "        # Update the network parameters\n",
    "        optimizer.step()\n",
    "        # Accumulate loss per batch\n",
    "        runningLoss += loss.data[0]    \n",
    "    avgTrainLoss = runningLoss/200\n",
    "    trainLoss.append(avgTrainLoss)\n",
    "    # Evaluating performance on test set for each epoch\n",
    "    net.train(False) # For testing\n",
    "    inputs = TestImages/255\n",
    "    if use_gpu:\n",
    "        inputs = Variable(inputs.cuda())\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        predicted = predicted.cpu()\n",
    "    else:\n",
    "        inputs = Variable(inputs)\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    total += TestLabels.size(0)\n",
    "    correct += (predicted == TestLabels).sum()\n",
    "    avgTestAcc = correct/60.0\n",
    "    testAcc.append(avgTestAcc)\n",
    "        \n",
    "    # Plotting Loss vs Epochs\n",
    "    fig1 = plt.figure(1)        \n",
    "    plt.plot(range(epoch+1),trainLoss,'r--',label='train')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Loss')    \n",
    "    # Plotting testing accuracy vs Epochs\n",
    "    fig2 = plt.figure(2)        \n",
    "    plt.plot(range(epoch+1),testAcc,'g-',label='test')        \n",
    "    if epoch==0:\n",
    "        plt.legend(loc='upper left')\n",
    "        plt.xlabel('Epochs')\n",
    "        plt.ylabel('Testing accuracy')  \n",
    "    \n",
    "    epochEnd = time.time()-epochStart\n",
    "    print('At Iteration: {:.0f} /{:.0f}  ;  Training Loss: {:.6f} ; Testing Acc: {:.3f} ; Time consumed: {:.0f}m {:.0f}s '\\\n",
    "          .format(epoch + 1,iterations,avgTrainLoss,avgTestAcc*100,epochEnd//60,epochEnd%60))\n",
    "end = time.time()-start\n",
    "print('Training completed in {:.0f}m {:.0f}s'.format(end//60,end%60))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}