ComputeDIScoreCNNVer04.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ComputeDIScoreCNNVer04.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/preritt/CS231n-/blob/master/ComputeDIScoreCNNVer04.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "KbUrxd4wz4UM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import keras\n",
        "from matplotlib import pyplot as plt\n",
        "import numpy as np\n",
        "import gzip\n",
        "%matplotlib inline\n",
        "from keras.models import Model\n",
        "from keras.optimizers import RMSprop\n",
        "from keras.layers import Input,Dense,Flatten,Dropout,merge,Reshape,Conv2D,MaxPooling2D,UpSampling2D,Conv2DTranspose\n",
        "from keras.layers.normalization import BatchNormalization\n",
        "from keras.models import Model,Sequential\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "from keras.optimizers import Adadelta, RMSprop,SGD,Adam\n",
        "from keras import regularizers\n",
        "from keras import backend as K\n",
        "from keras.utils import to_categorical\n",
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PdBYiAy_TR3t",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2dhbUZ2fmgml",
        "colab_type": "code",
        "outputId": "6e5866f5-276c-422c-cd02-64049cb0982e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        }
      },
      "cell_type": "code",
      "source": [
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "\n",
        "mnist = input_data.read_data_sets(\"MNIST_data/\", reshape=False)\n",
        "X_train, y_train           = mnist.train.images, mnist.train.labels\n",
        "X_validation, y_validation = mnist.validation.images, mnist.validation.labels\n",
        "X_test, y_test             = mnist.test.images, mnist.test.labels\n",
        "\n",
        "assert(len(X_train) == len(y_train))\n",
        "assert(len(X_validation) == len(y_validation))\n",
        "assert(len(X_test) == len(y_test))\n",
        "\n",
        "print()\n",
        "print(\"Image Shape: {}\".format(X_train[0].shape))\n",
        "print()\n",
        "print(\"Training Set:   {} samples\".format(len(X_train)))\n",
        "print(\"Validation Set: {} samples\".format(len(X_validation)))\n",
        "print(\"Test Set:       {} samples\".format(len(X_test)))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
            "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
            "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
            "\n",
            "Image Shape: (28, 28, 1)\n",
            "\n",
            "Training Set:   55000 samples\n",
            "Validation Set: 5000 samples\n",
            "Test Set:       10000 samples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "pq0Y_8Uoz4US",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from keras.utils.vis_utils import plot_model\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hggBv2Vpz4UV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Define function to load the data and conver to array format (#images,dim1,dim2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LKr7oBMfz4Uc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Extract images"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ajN_BPvlTGEi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d0376c5c-3012-48fc-b4eb-0693e2354747"
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "# Pad images with 0s\n",
        "X_train      = np.pad(X_train, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "X_validation = np.pad(X_validation, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "X_test       = np.pad(X_test, ((0,0),(2,2),(2,2),(0,0)), 'constant')\n",
        "    \n",
        "print(\"Updated Image Shape: {}\".format(X_train[0].shape))"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Updated Image Shape: (32, 32, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "PYZVx_TNz4U4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Create dictionary of target classes\n",
        "label_dict = {\n",
        " 0: 'A',\n",
        " 1: 'B',\n",
        " 2: 'C',\n",
        " 3: 'D',\n",
        " 4: 'E',\n",
        " 5: 'F',\n",
        " 6: 'G',\n",
        " 7: 'H',\n",
        " 8: 'I',\n",
        " 9: 'J',\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "SZGysJCjz4U6",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Explore some images"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YXNfimX1z4U9",
        "colab_type": "code",
        "outputId": "219b8947-3049-4c7b-e3b2-6faae6ed79bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 209
        }
      },
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=[5,5])\n",
        "\n",
        "# Display the first image in training data\n",
        "plt.subplot(121)\n",
        "curr_img = np.reshape(X_train[10], (32,32))\n",
        "curr_lbl = y_train[10]\n",
        "plt.imshow(curr_img, cmap='gray')\n",
        "plt.title(\"(Label: \" + str(label_dict[curr_lbl]) + \")\")\n",
        "\n",
        "# Display the first image in testing data\n",
        "plt.subplot(122)\n",
        "curr_img = np.reshape(X_test[10], (32,32))\n",
        "curr_lbl = y_test[10]\n",
        "plt.imshow(curr_img, cmap='gray')\n",
        "plt.title(\"(Label: \" + str(label_dict[curr_lbl]) + \")\")"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, '(Label: A)')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATwAAACuCAYAAACr3LH6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEUlJREFUeJzt3XuwXWOax/HvTy6E0JEglSAi2mhi\nXFOki4wMMXRThFKkh5GeoeKSUa1NEBnjUkaLLtcyJIKQKvoiGIIm45IMwaTpcU0inVDCSYcwEuI2\nJfHOH3sls9/VOfvsffY15/19qnad9ay19l5Pcp56ztrv3utdCiFgZpaCzZqdgJlZo7jhmVky3PDM\nLBlueGaWDDc8M0uGG56ZJSPphifpGknnV/kagyUFSd0b+dx2Xu96SefU4rWseq6v1pNsw5O0PXA6\ncHsWj5TU1tysyiNprqRVkjbPbboOmCSpZzPysv/n+mpNyTY84KfA70IIXzc7kUpIGgyMAAJwXPG2\nEMIK4O38emuKn+L6ajkpN7wfAf9Zzo6SjpH0qqTPJX0g6YqN7PYPkv4kaYWkCUXP3UzSREnvSPof\nSfdL6ltF3qcD/wXcA4zdyPa5wDFVvL7VhuurBaXc8P4SWFzmvl9SKIQ+FH7Z50gandvnr4Hdgb8B\nLpY0Klt/HjAaOAwYCKwCbt3YQbLCfayDXE4H7sseR0nqn9u+CNi3nH+U1ZXrqxWFEJJ8AN8CPyiK\nRwJtZT73JuDGbHkwhdP/4tf6JXBXtrwIOKJo24Ds2N2Lntu9zOMemj13uyx+G/h5bp8jgXeb/f+b\n+sP11ZqPlM/wVgFbl7OjpIMlzZH0saTPgLOB7XK7fVC0vIzCX1uAXYB/l7Ra0moKBboOyP/lLMdY\n4D9CCJ9k8a/487cdWwOrO/HaVluurxaUcsN7A/iLMvf9FTAL2DmE8D1gKqDcPjsXLQ8C/pQtfwD8\nKITQp+ixRQhheSXJSuoFnAwcJulDSR8CPwf2lVT8FmNP4PVKXtvqwvXVglJueL+jMO4RkbRF7iEK\nf9U+DSF8I+kg4G838nr/ImlLSUOBvwd+m62fClwtaZfs9beXdHwn8h1N4S/3XsB+2WNP4HkK4y7r\nHQY80YnXt9pyfbWiZr+nbtaDwluGNqBXFo+kMN6Rf3wfOInC24g1wGPAvwH3hniMZRyFv7ofAhcV\nHWcz4AIKA9hrgHeAX+Se2z2LJwFPtJPvk8D1G1l/cnbM7hTGb9qAns3+/0394fpqzYeyf1SSJP0C\nWBlCuKnZudSCpOuBd0IItzU7F3N9taKkG56ZpSXlMTwzS4wbnpklo6qGJ+loSYslLZU0sVZJmYHr\ny2qv02N4kroBf6Twzes24GXgJyGEhbVLz1Ll+rJ6qGaerIOApSGEdwEk/QY4Hmi3ICX5E5J0fRJC\n2L6C/V1fVomy6quat7Q7El/u0patM9uYZRXu7/qySpRVXzWZCbUUSeMofGnSrOZcX1aJahrecuLr\n+3bK1kVCCNOAaeC3HFYR15fVXDVvaV8Gdpe0azbl8xgKF0Cb1YLry2qu02d4IYS1kv4RmA10A6aH\nEBbULDNLmuvL6qGhl5b5LUfS/hBCGFbPA7i+klZWfflKCzNLhhuemSXDDc/MkuGGZ2bJcMMzs2S4\n4ZlZMtzwzCwZbnhmlgw3PDNLhhuemSXDDc/MkuGGZ2bJcMMzs2S44ZlZMtzwzCwZbnhmlgw3PDNL\nRt3vWmZmnXPooYdG8UsvvRTFe+yxx4blY489Ntp2zDHHRPHjjz9e8lgvvvhiFM+bN6/sPDclPsMz\ns2S44ZlZMtzwzCwZHsOrgKQoHjNmTBRffvnlUVw8xlKpxYsXR/ERRxwRxR999FEUr127ttPHsubY\nZpttovi+++6L4sMPPzyKv/766yju2bPnhuXevXuXPNaIESNKbs+/9ldffRXF55xzThQ/8MADJV+v\nVfkMz8yS4YZnZslwwzOzZCiExt2sfVO7M/xmm8V/D8aPHx/FN998c8nnf/fddxuW82Mi3bp1i+Je\nvXpVlNuCBQuieNSoUVGcH+NrAWXdGb4am1p9TZkyJYrPOuusip6/aNGiDcsff/xxtO3zzz8v+dz8\neHT+e3t5a9asieL8mOAbb7xR8vkNUFZ9+QzPzJLhhmdmyXDDM7NkeAyvhHHjxkXx1KlTS+6/bt26\nKL7iiis2LF999dXRtkGDBkXxhRdeGMVnn312FOfH/PLyY3qHHHJIFHc0ptMAyY/hDR06NIrnzp0b\nxf369Yvitra2KD799NOjeOnSpRuWV69eHW374osvSuaSH5++7LLLovjSSy+N4nz9PfTQQ1F85pln\nRvGqVatKHr8OPIZnZlasw4YnabqklZLeKlrXV9JTkpZkP7etb5rWlbnGrFHKOcO7Bzg6t24i8EwI\nYXfgmSw266x7cI1ZA3R4LW0I4TlJg3OrjwdGZsszgLnAxTXMqyny4xQjR46s6PmTJ0+O4vy4XbH3\n338/is8777wofu6556L4pptuiuIBAwZEcX58aMstt4ziFhjDa1cqNbb11ltHcX7MLj+efu2110Zx\nfsyvGsXfEYV4vBni63QBJkyYEMUnnHBCFE+fPj2KO5p/r1k6O4bXP4SwIlv+EOhfo3zM1nONWc1V\nPVtKCCGU+nRM0jhgXHvbzTpSqsZcX1aJzp7hfSRpAED2c2V7O4YQpoUQhtX7KwnW5ZRVY64vq0Rn\nz/BmAWOBydnPR2qWURPtsMMOUZyf7y4v/923O++8s2a5zJw5M4rPP//8KM6P4XVBXa7GNt9885Lb\nZ8yYEcW33nprPdMpadKkSVF8yimnRPGuu+4axSeeeGIUb7JjeJJ+DbwE7CGpTdIZFIrwSElLgFFZ\nbNYprjFrlHI+pf1JO5uOaGe9WUVcY9YovtLCzJLhe1oUGT16dMnt3377bRRfdNFFUbxs2bKa57Te\nqaeeGsX5e5T27x9/a2Ps2LFRfN1110Vx/rpfq7+rrrqq5Pb58+c3KJPKzZ49O4rz13oPHz68kel0\nms/wzCwZbnhmlgw3PDNLRtJjePlrGy+44IKS+7/33ntR/MQTT9Q6pbKPnf/OVn488Zprronihx9+\nOIrz97212hsyZEgUDxw4MIo/++yzKH7zzTfrnlNnPfvss1GcH8PbVPgMz8yS4YZnZslwwzOzZCQ9\nhpef82u33XZrUiaVW7hwYUX75+952tF4pVXvtNNOi+L8mN6DDz4YxS+++GLdc0qdz/DMLBlueGaW\nDDc8M0tG0mN4lVq+fHmzU7BNSH4+xfz37m6++eZGpmP4DM/MEuKGZ2bJSPotbf5rAx25++6765SJ\npeDtt9+O4nnz5jUpk3T5DM/MkuGGZ2bJcMMzs2QkPYaXv9WcWTW22mqrKO7Ro0eTMrH2+AzPzJLh\nhmdmyXDDM7NkJD2GZ1ZLJ598chTnpxv75JNPGplOTR133HElt69du7ZBmVTHZ3hmlgw3PDNLhhue\nmSXDY3glfPnll1H8/vvvNymT6vm2jFaJAw88MIqPPfbYkvtPmjSpnunUjM/wzCwZHTY8STtLmiNp\noaQFkn6Wre8r6SlJS7Kf29Y/XetqXF/WSOWc4a0F/imEsBcwHBgvaS9gIvBMCGF34JksNquU68sa\npsMxvBDCCmBFtrxG0iJgR+B4YGS22wxgLnBxXbJskvxtHLfZZpsmZQKDBg2K4gkTJlT0/JkzZ9Yy\nnZpJub5aSX7MLn8bzz59+kTxCy+8EMWzZ8+uT2I1VtEYnqTBwP7AfKB/VqwAHwL9a5qZJcf1ZfVW\n9qe0knoDDwLnhxA+l7RhWwghSArtPG8cMK7aRK1rc31ZI5R1hiepB4VivC+E8FC2+iNJA7LtA4CV\nG3tuCGFaCGFYCGFYLRK2rsf1ZY3S4RmeCn9q7wIWhRBuKNo0CxgLTM5+PlKXDOvo9ddfL7k9P59Z\n/rtGjz76aM1zas+9994bxXvvvXfJ/SdOjMf487cIbBVdqb7ee++9KF6zZk1zEilDt27dojg/JnzK\nKadEcf4Wpfn9N5Vract5S3sI8HfAm5Jey9ZNolCI90s6A1gGnNzO881KcX1Zw5TzKe08QO1sPqK2\n6VhqXF/WSL7SwsySkfS1tLNmzapo/759+9Ypkz93ySWXRPHBBx9ccv/8tbK33357FK9bt642iVm7\n5syZE8X5ca/89zi32267KK7lfHn77LNPFJ977rlRfMABB0TxsGGlP/PJ38N5/vz5VWTXPD7DM7Nk\nuOGZWTLc8MwsGUmP4X3zzTdRvGDBgigeOnRoFA8ZMiSKb7nllii+8cYbo/jdd98tO5dRo0ZF8ZVX\nXhnF3bvHv6r8mN1RRx0Vxa36vbuU7bnnnlH85JNPRvGKFSuoleHDh0dxv379Su6fHz/Mj2+//PLL\ntUmsyXyGZ2bJcMMzs2S44ZlZMhTCRiehqM/B2pnxolX07x/PQPT0009HcX5ML2/p0qVRfNttt7W7\n79ixY6M4fw/T3r17lzzW+PHjo3jKlCkl928Bf6j3Bf6tVl8nnHBCFF966aVRvP/++zcsl++++y6K\nP/300yi+4YYbonjy5Ml1z6nGyqovn+GZWTLc8MwsGW54ZpYMj+GVcNJJJ0Xx5ZdfHsUdjelVY8mS\nJVGc/55d/h65+TGaFpTcGF7ewIEDozj/PbyO5jisxB133BHFr776ahRPnTq1ZsdqER7DMzMr5oZn\nZslwwzOzZHgMrwL561nz39s766yzonjEiBEblp9//vmSrz19+vQobmtri+JN5Z4BJSQ/hmd15TE8\nM7Nibnhmlgw3PDNLhsfwrFE8hmf15DE8M7Nibnhmlgw3PDNLhhuemSXDDc/MkuGGZ2bJcMMzs2S4\n4ZlZMtzwzCwZHTY8SVtI+r2k1yUtkHRltn5XSfMlLZX0W0k965+udTWuL2ukcs7w/hc4PISwL7Af\ncLSk4cC1wI0hhO8Dq4Az6pemdWGuL2uYDhteKPgiC3tkjwAcDjyQrZ8BjK5Lhtalub6skcoaw5PU\nTdJrwErgKeAdYHUIYf2slG3AjvVJ0bo615c1SlkNL4SwLoSwH7ATcBDwg3IPIGmcpFckvdLJHK2L\nc31Zo1T0KW0IYTUwB/gh0EfS+jnPdwKWt/OcaSGEYfWeGsg2fa4vq7dyPqXdXlKfbLkXcCSwiEJh\nrr9x61jgkXolaV2X68saqXvHuzAAmCGpG4UGeX8I4TFJC4HfSPpX4FXgrjrmaV2X68saxjMeW6N4\nxmOrJ894bGZWrJy3tLX0CbAM2C5bbkWtnBtsuvnt0oBjbwr1Ba2dXyvnBlXWV0Pf0m44qPRKq36q\n1sq5gfPbVHIopZXza+XcoPr8/JbWzJLhhmdmyWhWw5vWpOOWo5VzA+dXjlbIoZRWzq+Vc4Mq82vK\nGJ6ZWTP4La2ZJaOhDU/S0ZIWZ5M6TmzksdvJZ7qklZLeKlrXV9JTkpZkP7dtYn47S5ojaWE2OebP\nWiXHVpy40/VVcX7p1VcIoSEPoBuFaX+GAD2B14G9GnX8dnL6K+AA4K2idb8EJmbLE4Frm5jfAOCA\nbHlr4I/AXq2QIyCgd7bcA5gPDAfuB8Zk66cC57i+XF+tUl+N/Af8EJhdFF8CXNKsX3ZRHoNzBbkY\nGFBUEIubnWNRbo9QuLi+pXIEtgT+GziYwpdCu2/sd17nHFxf1efa5eurkW9pdwQ+KIpbdVLH/iGE\nFdnyh0D/ZiaznqTBwP4U/tK1RI4tNnGn66sKqdSXP7QoIRT+jDT9Y2xJvYEHgfNDCJ8Xb2tmjqGK\niTvN9dWRetRXIxvecmDnorjdSR2b7CNJAwCynyubmYykHhSK8b4QwkPZ6pbKMXRi4s46cH11Qmr1\n1ciG9zKwe/YpS09gDDCrgccv1ywKE05CkyeelCQK88AtCiHcULSp6Tm24MSdrq8KJVlfDR58/DGF\nT4LeAf65mQOhWT6/BlYA31IYDzgD6Ac8AywBngb6NjG/Qym8nXgDeC17/LgVcgT2oTAx5xvAW8Bl\n2fohwO+BpcBMYHPXl+urVerLV1qYWTL8oYWZJcMNz8yS4YZnZslwwzOzZLjhmVky3PDMLBlueGaW\nDDc8M0vG/wFalV+8Fiin3gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 360x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "a_EONBUbz4VK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# make sure data is in float32 format\n",
        "X_train = X_train/np.max(X_train)\n",
        "X_validation = X_validation/np.max(X_validation)\n",
        "X_test = X_test/np.max(X_test)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tleGG-epz4Vp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# model parameters\n",
        "batch_size = 16\n",
        "epochs = 1\n",
        "inChannel = 1\n",
        "x, y = X_train.shape[1],X_train.shape[2]\n",
        "input_img = Input(shape = (x, y, inChannel))\n",
        "num_classes = 10"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "67YaEkrs1e9u",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def getScoresAndLayersDense(input_img,instance_number = 'First'):\n",
        "  appended_name = instance_number\n",
        "  conv1_1 = Conv2D(6, (3, 3), activation='relu', padding='same',name = appended_name + '_conv1_1')(input_img) #28 x 28 x 32\n",
        "  conv1_1 = BatchNormalization(name = appended_name + '_bn1_1')(conv1_1)\n",
        "\n",
        "  pool1_1 = MaxPooling2D(pool_size=(2, 2), name = appended_name + '_maxPool1_1')(conv1_1) #14 x 14 x 32\n",
        "  conv1_2 = Conv2D(16, (3, 3), activation='relu', padding='same',name = appended_name + '_conv1_2')(pool1_1) #28 x 28 x 32\n",
        "  conv1_2 = BatchNormalization(name = appended_name + '_bn2_1')(conv1_2)\n",
        "\n",
        "  pool1_2 = MaxPooling2D(pool_size=(2, 2), name = appended_name + '_maxPool1_2')(conv1_2) #14 x 14 x 32 \n",
        "  \n",
        "  flat = Flatten()(pool1_2) # 784*1\n",
        "  hidden1 = Dense(300, activation='relu',name = appended_name + '_Hidden1')(flat)\n",
        "  hidden2 = Dense(100, activation='relu',name = appended_name + '_Hidden2')(hidden1)\n",
        "  scores = Dense(num_classes, activation='softmax',name = appended_name + '_scores')(hidden2)\n",
        "  return conv1_1,hidden1,hidden2,scores"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Twq9agX7z4WR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "conv1_1, hidden1,hidden2,scores = getScoresAndLayersDense(input_img,'First')\n",
        "classifier = Model(inputs=[input_img], outputs=[scores])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "k-AK2DK64ha9",
        "colab_type": "code",
        "outputId": "d2394f98-d979-470a-fdd7-1a9e3d696ea8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        }
      },
      "cell_type": "code",
      "source": [
        "classifier.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(),metrics=['accuracy'])\n",
        "classifier.summary()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         (None, 32, 32, 1)         0         \n",
            "_________________________________________________________________\n",
            "First_conv1_1 (Conv2D)       (None, 32, 32, 6)         60        \n",
            "_________________________________________________________________\n",
            "First_bn1_1 (BatchNormalizat (None, 32, 32, 6)         24        \n",
            "_________________________________________________________________\n",
            "First_maxPool1_1 (MaxPooling (None, 16, 16, 6)         0         \n",
            "_________________________________________________________________\n",
            "First_conv1_2 (Conv2D)       (None, 16, 16, 16)        880       \n",
            "_________________________________________________________________\n",
            "First_bn2_1 (BatchNormalizat (None, 16, 16, 16)        64        \n",
            "_________________________________________________________________\n",
            "First_maxPool1_2 (MaxPooling (None, 8, 8, 16)          0         \n",
            "_________________________________________________________________\n",
            "flatten_2 (Flatten)          (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "First_Hidden1 (Dense)        (None, 300)               307500    \n",
            "_________________________________________________________________\n",
            "First_Hidden2 (Dense)        (None, 100)               30100     \n",
            "_________________________________________________________________\n",
            "First_scores (Dense)         (None, 10)                1010      \n",
            "=================================================================\n",
            "Total params: 339,638\n",
            "Trainable params: 339,594\n",
            "Non-trainable params: 44\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7SYqs58hz4Wy",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "plot_model(classifier, to_file=\"LeNEt300-100.png\", show_shapes=True, show_layer_names=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1IijEibsYB52",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_Y_one_hot = to_categorical(y_train)\n",
        "validation_Y_one_hot = to_categorical(y_validation)\n",
        "test_Y_one_hot = to_categorical(y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": true,
        "id": "ExpZ4568z4XA",
        "colab_type": "code",
        "outputId": "712d353b-6c2d-452e-8556-ac789ab222a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "cell_type": "code",
      "source": [
        "# train the model\n",
        "# classifier_train = classifier.fit(train_X, train_ground, batch_size=batch_size,epochs=1,verbose=1,validation_data=(valid_X, valid_ground))\n",
        "classify_train = classifier.fit(X_train, train_Y_one_hot, batch_size=64,epochs=5,verbose=1,validation_data=(X_validation, validation_Y_one_hot))\n"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 55000 samples, validate on 5000 samples\n",
            "Epoch 1/5\n",
            "55000/55000 [==============================] - 12s 218us/step - loss: 0.1405 - acc: 0.9567 - val_loss: 0.0867 - val_acc: 0.9746\n",
            "Epoch 2/5\n",
            "55000/55000 [==============================] - 11s 201us/step - loss: 0.0530 - acc: 0.9833 - val_loss: 0.1046 - val_acc: 0.9694\n",
            "Epoch 3/5\n",
            "55000/55000 [==============================] - 11s 200us/step - loss: 0.0358 - acc: 0.9881 - val_loss: 0.0679 - val_acc: 0.9786\n",
            "Epoch 4/5\n",
            "55000/55000 [==============================] - 11s 208us/step - loss: 0.0264 - acc: 0.9913 - val_loss: 0.0588 - val_acc: 0.9850\n",
            "Epoch 5/5\n",
            "55000/55000 [==============================] - 11s 201us/step - loss: 0.0200 - acc: 0.9933 - val_loss: 0.0497 - val_acc: 0.9876\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "gW-wbrPT76J0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "get_hidden1_output_function = K.function([classifier.layers[0].input],\n",
        "                                  [classifier.get_layer('First_Hidden1').output])   #get_layer\n",
        "hidden1_output = get_hidden1_output_function([X_test])[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mKAEb1HfLa3q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "get_cnn1_output_function = K.function([classifier.layers[0].input],\n",
        "                                  [classifier.get_layer('First_bn1_1').output])   #get_layer\n",
        "cnn1_output = get_cnn1_output_function([X_test])[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JqiaOhy376Dx",
        "colab_type": "code",
        "outputId": "6a6e305e-d437-4fd5-a5fb-32cb0b79ed3b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "hidden1_output.shape"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000, 300)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "metadata": {
        "id": "-rtO4BBkLnHm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0c124423-d721-4ff6-90a5-14446ba32e08"
      },
      "cell_type": "code",
      "source": [
        "cnn1_output.shape"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000, 32, 32, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "metadata": {
        "id": "AnX5N3ca75-A",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import PCA\n",
        "from sklearn.manifold import TSNE"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "N0aCwjcX753q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "pca1 = PCA(n_components=2)\n",
        "F1_PCA = pca1.fit_transform(hidden1_output) #test_data, test_Y_one_hot\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tG8FcCJr-n_m",
        "colab_type": "code",
        "outputId": "15eacf8b-837a-4e8d-915c-341a671f5a26",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 269
        }
      },
      "cell_type": "code",
      "source": [
        "colors = ['C0', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9']\n",
        "for i in range(200):\n",
        "    plt.scatter(F1_PCA[i,0],F1_PCA[i,1],c=colors[y_test[i]])"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXuYFHeZ7z/vXJgZLhlCgDAMEAKS\nhERHw3KCEdFNRkkUObhesqw5a9Z1F90Tj8iz8WwuxzgbTzZxE0Vc4xM5z7pLfOIix0SRxWyCwFEE\nQ5aEZJQQwiUXQmYCBBhuM8NcfueP7hr6UtVd3V3VVdX1fp5nnnT/urrqHR2+9av39/29rxhjUBRF\nUSqfqqADUBRFUcqDCr6iKEpMUMFXFEWJCSr4iqIoMUEFX1EUJSao4CuKosQEFXxFUZSYoIKvKIoS\nE1TwFUVRYkJN0AGkMnbsWDN16tSgw1AURYkUzz777FFjzLh8x4VK8KdOncqOHTuCDkNRFCVSiMhr\nbo7TlI6iKEpMUMFXFEWJCSr4iqIoMUEFX1EUJSao4CuKosQEFXxFUZSYoIKvKIoSE1TwFUVRYkKo\nNl4piuINP995iAee3MObJ7qZOLqBr95wOR+/ujnosJSAUcFXlArj5zsPccfjv6e7bwCAQye6uePx\n3wOo6MecklM6IlIvIs+IyAsisktE/j45fqmIbBeRfSLyExEZVnq4ipKf9QfWM/+n82lZ1cL8n85n\n/YH1QYdUVh54cs+Q2Ft09w3wwJN7AopICQte5PB7geuNMe8G3gPcKCLvBb4JLDfGvAM4Dnzeg2sp\nSk7WH1hP27Y2Os50YDB0nOmgbVtbrET/zRPdBY0r8aFkwTcJTiff1iZ/DHA98NPk+Crg46VeS1Hy\nseK5FfQM9KSN9Qz0sOK5FQFFVH4mjm4oaFyJD564dESkWkSeBw4DG4D9wAljTH/ykDcA2+ShiCwR\nkR0isuPIkSNehKPEmM4znQWNVyJfveFyGmqr08Yaaqv56g2XBxSREhY8EXxjzIAx5j3AJOAa4IoC\nvrvSGDPbGDN73Li85ZwVJScTRkwoaLwS+fjVzdz3iXfRPLoBAZpHN3DfJ96lC7aKty4dY8wJEdkM\nXAuMFpGa5Cx/EnDIy2spih1LZy2lbVtbWlqnvrqepbOWBhhV+fn41c0q8EoWXrh0xonI6OTrBuDD\nwG5gM/Cp5GG3AGtLvZai5GPBtAW0va+NphFNCELTiCba3tfGgmkLgg5NUQLHixl+E7BKRKpJ3EDW\nGGP+XUReBFaLyP8GdgL/7MG1FCUvC6YtUIFXFBtKFnxjTDtwtc34ARL5fEVRFCUEaC0dRVGUmKCC\nrygxIO67j5UEWktHUSoca/ex5Vyydh8Dnq11aLG2aKAzfEWpcPzefWwVazt0ohvD+WJtP9+pTuyw\noYKvKBWO37uPtVhbdFDBV5QKx+/dx1qsLTqo4CtKhbN01lLqq+vTxrzcfazF2qKDCr6iVDh+7z7W\nYm3RQV06ihID/Nx9bLlx1KUTflTwFU9Yf2A9K55bQeeZTiaMmMDSWUu1vEGM0GJt0UAFXymZcvi8\nlfyoF17Jh+bwlZIp1Oetuz69R73wihtU8JWSKcTnrT1n/UG98IobVPCVkinE5609Z/1BvfCKG1Tw\nlZIpxOetPWf9oRgv/M93HmLu/Zu49Pb1zL1/k6Z/YoAKvlIUqXn4Fc+tYNE7FrnyeWvPWX8o1Auv\nOf94ooKvFIxdHn7tvrUsnbWU9lvaeepTTzm6c/ze9RlXCm1crjn/eKK2TKVgcuXh89kwrc/Vs+89\nhXjhNecfT1TwlYIpNQ+vPWeDZ+LoBg7ZiLvWv6lsNKWjFExY8/CV6O/3a2FV69/EExV8pWDCmIf3\n0t8flhtHKQur+W4Uheb8lcpAjDFBxzDE7NmzzY4dO4IOQ3FBWGrnWHF0nOmw/bxpRBNPfeop2+/Y\nxZ5ZJgISNzMvq0u6Ze79m2zTLs3JsglOZRSsG0XqomxDbbUKegUjIs8aY2bnPU4FX4kqduKciSC0\n39Ke8zupgj7/p/Ntbx52Nw6/ufT29Tj962yorXYU9Fw3iq23X+9TtEqQuBV8TekokcXOLZRJ5rpC\nvp2+YdoY5rSAWi2S01JZigNHN2NVNir4SmTJJ8J26wr5BN2PBeliRdRpYXXA4ancEvRiO1DpZqzK\nRwVfiSz5RLiuus71d6xxrxekSxFRp4XV5jyCXqwDRzdjVT7qw1ciy9JZS3Pm8LvOdWXV5bf7Tqqg\ne70xLJeIullAddpMZbcoawl6sR2odDNW5VPyoq2ITAYeAS4GDLDSGLNCRMYAPwGmAq8CNxljjuc6\nly7aKoWS6rgREQbNYNYxmQuubh1GXjiRnBZeBXjl/uJdP340O7n6nqc4frYvazzXYq82XQkHbhdt\nvZjh9wN/a4x5TkRGAc+KyAbgL4CNxpj7ReR24Hbg7zy4nqIMkbprt2VVi+0xmXl7Nzt9veri5deO\nVq9bCv585yFO9/RnjddWS94CbNaThpWusuJTwkfJOXxjTIcx5rnk61PAbqAZWASsSh62Cvh4qddS\nlFx4ueDqVd3+qOxofeDJPfQNZj+LjBhWowXYKghPF21FZCpwNbAduNgYYxmaO0mkfBTFN7xccHXa\nyOU07kRUdrQ65em7urNTPPm+ozn/8OLZoq2IjAQeA75ijDkpIkOfGWOMiNguFojIEmAJwJQpU7wK\nR4khXi64VkmV7XpAlRQ+R/I6/eIHxaSetABb9PBE8EWkloTYP2qMeTw5/JaINBljOkSkCThs911j\nzEpgJSQWbb2IR4kvXlXitBP7XONR56s3XJ7T+ePVd5RgKTmlI4mp/D8Du40x30756BfALcnXtwBr\nS72WopSLphFNBY1HnWJST1FJVynn8cKW+X5gC/B7wJr+3Ekij78GmAK8RsKWeSzXudSWWfmEpeha\nPsJURE1R8lE2W6Yx5rckbMV2tJZ6fqVy8MrqWA60M5dSiWi1TKVshKkSpaJUElotUwkdYapEqShx\nRAVfKRthbY2oKHFBi6cpZSNf4TKlNNzWtdH6N/FFBV8pG7oQ6h9u69po/Zt4o4u2ilIBuG1rqO0P\nKxNdtFWUGOG2ro3Wv4k3KviKUgHka2totVl0ep7X+jfxQAVfUSqAXGWYU9ss2qH1b+KDLtoqSgWQ\nq63h3Ps3ZdWtt2hWl06sUMFXlAohswyzlcZxmtkL6EJtzFDBVyqCqBRlKxeZ9ks7NG8fP1TwlcgT\npaJs5cKu/WAqmrePJ7poq0Qer/rPVhK5bJaZdeut1M+lt69n7v2b+PnOQ+UKUykzOsNXIo8WZcvG\nqf1g5gYr3XkbL3SGr0QeLcqWTS6bZip2qZ/uvgEeeHKP7zEq5UcFX4k8S2ctpb66Pm0s7kXZ3LYf\n1J238UJTOkrk0aJs9mTaNO1wSv2og6cyUcFXKoIF0xbEXuCL4as3XJ5l31QHT+Wigq8oMSbXDl2l\n8lDBV1zT3t7Oxo0b6erqorGxkdbWVlpaWoIOSykRN6kfpTJQwVdc0d7ezrp16+jr6wOgq6uLdevW\nAajoK0pEUJeO4oqNGzcOib1FX18fGzduDCgiRVEKRQVfcUVXV1dB44qihA9N6ShA/vx8Y2Ojrbg3\nNjaWM0xP0WbeStzQGb4ylJ+3BN3Kz7e3tw8d09raSm1tbdr3amtraW1tLWusXpHaFMRwvqSA1pFR\nKhkVfMVVfr6lpYWFCxcOzegbGxtZuHBhZBdstaSAEkc8SemIyA+BjwGHjTHvTI6NAX4CTAVeBW4y\nxhz34nqKt7jNz7e0tERW4DPRkgJKHPFqhv+vwI0ZY7cDG40xM4CNyfdKCHHKw0c5P5+PfE2/FaUS\n8UTwjTG/AY5lDC8CViVfrwI+7sW1FO+ptPy8G9xWk1SUSsJPl87FxpiO5OtO4GIfr6WUgJWmyXTp\nACxfvrwid9ZqSQEljogxxpsTiUwF/j0lh3/CGDM65fPjxpgLbb63BFgCMGXKlD967bXXPIlHKY3M\nnbWQmPVHeaFWUSoVEXnWGDM733F+unTeEpGmZDBNwGG7g4wxK40xs40xs8eNG+djOEoh6M5aRak8\n/BT8XwC3JF/fAqz18VqKx+jOWkWpPLyyZf4b8MfAWBF5A/g6cD+wRkQ+D7wG3OTFtZTyUIk7a5Xw\ncWbnYU4++SoDJ3qpHl3HBTdMZcTV44MOq2LxRPCNMX/m8FHl2jwqnNbWVtscfiU7d5TycmbnYU48\nvhfTNwjAwIleTjy+F0BF3yd0p61iS6XtrFXCx8knXx0SewvTN8jJJ18NJqAYoMXTFEcqaWetEj4G\nTvQWNK6Ujgq+UlY6OtdyYP+D9PR2UF/XxLTpt9E0YVHez5TKo3p0na24V4+uCyCaeKCCr5SNjs61\nvPTSXQwOJurV9PS+yUsv3TX0udNnQYm+3oD85YIbpqbl8AGktooLbpgaXFAVjgq+UjYO7H9wSNAt\nBge7ObD/waHXdp8FIbK5bk4q+t5gLcyqS6d8qOArZaOnt6Og8Xyf+Umum5MKvneMuHq8CnwZUZeO\nUjbq65ocx3N9FgTF3JwUJeyo4CtF0dG5lq1b57Fx0zvYunUeHZ35N1JPm34bVVXp5YerqhqYNv22\nnJ8FQdhuQIriBZrSUQqm2Py29VmuhdCwLJJOm35b2u8Iwd6AFMULVPArkHwNyUullPx204RFWcdk\numGuvPJbgefJ3dycFCVqqOAHRNe6dRxe/h36OzqoaWpi/LKv0LhwYcnnzSxrbDUkBzwTfS/z22F2\nw9jdnBQlymgOPwC61q2j42t30//mm2AM/W++ScfX7qYrKcylUI6yxl7mt/NZNRVF8Q6d4QfA4eXf\nwfT0pI2Znh4OL/9OybN8r8oa50oL2eW3BwdrqK0tvCCqumEUpXzoDD8A+jvsxcxpvBC8aEhupYWs\nm4SVFmpvbwcSqY4Rw79Ab88IjIGenhG8vGcOGzacHDrGLeqGUZTyoTP8AKhpakqkc2zGS8WLssa5\n0kLWLH/r1n66uj4BwN5xzWyfdhWn6xpY1XGG+8Yf45MTxmSd1+6pQd0wilI+VPADYPyyr9DxtbvT\n0jpSX8/4ZV8p+dxODckLWbB1kxayXu8d18yvL7+a/urEn9LJYQ3ctucgr7/+OmzdPBTDjBkzeOGF\nF7IWkxcuXMgVV9yrbhhFKQMq+B7i1nljjfnh0oHSyxq76XZ1ySWdjL94G++vO8NCGcsaczPb5AMA\ndA8avv/2WW5OSQnt2LEj63zWU8OyZctU4JWyEecuWyr4HmE5b6xZu+W8ARxF3yuB95p8aaGOzrVM\nueQ3QKK07TiO8lc8DIYh0T9V15B1Xju6urp83zegKBZx77Kli7Yekct5EzXydbtKWCbT65jX0ctN\nPDr0fmRvutXSiYaGhpwLxJXK+gPrmf/T+bSsamH+T+ez/sD6oEOKBXHvsqUzfI/w03njhJ8z41xp\nISfL5EW8DUDN4ABzDuzKe43a2lqAvAvElcb6A+tp29ZGz0BigtBxpoO2bW0ALJi2IMDIKp+4d9lS\nwfcIP503dpRjR63TDaW+rome3uzf9W0uYlJdLTfXwNkTh0mV8draWt797nezd+/etPM9/vjjttcu\ndN9AlFjx3IohsbfoGehhxXMrVPB9Ju5dtlTwPcJP540dbqyTpZDrhuJkpfzgFXeyeMJVie/XV7l6\n+rCOyaSQfQNRo/NMZ0HjinfEvcuWCn4OCql347fzJis2j3bUOpHrhrJs2TIgd2Ext04hL/YNRI0J\nIybQcSY7LTZhxIQAookXce+yVVGC72VBskJdN9Z4uZw3bqyTpZDvhuJVYTHrpvDEE0/Q3Z14Yqip\nqag/yyyWzlqalsMHqK+uZ+mspQFGFR/i3GWrYlw6XhckC7vrprW1dWjR08LLmbEXJRoKob+/f+h1\nd3d3RTt1FkxbQNv72mga0YQgNI1oou19bZq/V3ynYqZSXhckC8J1Uwhe7KjNRSGplvUH1rPiuRV0\nnulkwogJLJ21tCDx8ns9IowsmLYgeIFvXwMb74GuN6BxErTeDS2FF8ArlThvhCo3FSP4Xgt0uV03\nxeA2T15MqsvtDcULi6Hf6xGKDe1rYN2XoS+58N51MPEeyir6cd8IVW4qRvC9Fuhyu278opi1CAs3\nNxQvLIZ+r0coNmy857zYW/R1J8bLKPi5NkKp4HuP7zl8EblRRPaIyD4Rud2v64xf9hWkvj792iUI\ndOPChTR94x5qJk4EEWomTqTpG/eEthyCE36vRThZCTvOdLB8+XLa2tpYvnx5zny83+sRpfBY5zFm\nb9tF0+bnmb1tF491Hgs6JG/oeqOwcZ+I+0aocuOr4ItINfAQ8BHgSuDPRORKP67lh0A3LlzIjE0b\nmbn7RWZs2ujqXF3r1rH3+lZ2z7ySvde3etLFqhScUlp9b77pyaKok5VweP9w1+US8pVyCIrHOo9x\n256DvNHbhwHe6O3j1t2vM/M37dEX/sZJhY37hNOGp7hshCo3fqd0rgH2GWMOAIjIamAR8KIfF8v0\nwluzWD9n5an58erGRgZOn4ak46SQ9Emx5Cuv4JTqOjt8OE96sDPXzmJYbaq56thVacfZLcI+1nmM\n+w50cKi3j+a6Wu74s1ts6+gHxX0HOugeNFnjxwcGuW3PQYBQxVsQrXen5/ABahsS42Uk7huhyo3f\nKZ1m4GDK+zeSY0OIyBIR2SEiO44cOVLSxfzsFevmegMnTgyJvYWfVs58nanAPtXVX11Ne0uL6163\nHZ1r2bp1Hhs3vYOtW+fR0bl26DM7i+GsI7OYcmZK1nlS8/R2s+fb9hwM1cz5UG+f42fdg4b7DoTD\nsVUULTfBwu9C42RAEv9d+N2yu3RGXD2e0Z+YMTSjrx5dx+hPzND8vU+IMdkzGM9OLvIp4EZjzF8l\n3/85MMcY8yW742fPnm3s6qa7Ze/1rfYLtxMnMmNTaU287Zwuh5d/x/Z6WYgwc7f3DzXLly93XOy0\ndsNCMs309TaGnz3L2eHDaW9p4fWplwx93tbW5niNjs61tmUUrrjiXseNV05xiQjGGBobG/mXWddx\nODmpe8dbB5nzyouM7O2mu344/+2jNwaezgGYvW0Xb+QQfQE6rntP+QJSFAdE5FljzOx8x/md0jkE\nTE55Pyk55gt+eeednC6Zi6FO+GXldGtnbFy4kC379hXlhDmw/8E0sQcYHOzmwP4HHQXfzsMPYE0u\nurq6uODN1/nwq7uHyihL8pjhPWc9LwJXLHdMa+K2PQdt0zoAzXW1tuOKElb8Fvz/BGaIyKUkhH4x\n8Bm/LuaXd97J6UJ1NQwM5Pyun1bOQuyMhWykSn2a6fle73k1TsGpRDJke/itmX0qf7z3eWoG0+14\n48YdYOqlz1NXd4aOzh9z/Hg9/QNdgbU9tPLz/2vvIY73p///3FAl3DEtPHsyooBusAoeX3P4xph+\n4EvAk8BuYI0xJn+h9CLx2ppp4fiEMDCQfb3aWmT06LJYOQuxM7p1wmSuS1Q7pNTr63KLXUtLC8uW\nLaOtrS1L7AFbsZ9x2dPU159BBGpqeugfOAEYenrf5KWX7kpbOygXn5wwht3z3sVDM6cwqa4WASbV\n1fLg5ZOju2AbANYGK8tuaW2wOrPzcMCRxQvfN14ZY34J/NLv64B/FSsdnxwmTjyfyy9DhcxMCi2v\n4GYjVebTzKi11XTdPIBJcclVVTUwbfptruN0ehJJZeqlz1Nd7fy0lC+N5DefnDBGBb4EdINVOKiY\nnbYWflSszLXrNujetKU2LM8k82lm+I5qAE4tGmDgIikqveKU00+lru5M3vPkSiOFikJq1ISkno0b\nXt7eye/W7uf0sV5Gjqnj2kXTuWyOu5LOYdpgFefUUsUJvh+Uu9Z9Ocn08V/V0sLkF15IO2b4jmou\neHNy0U6nzCeRhoYGzp07x0DK+kdv7wjq63OLfr40UigopEZNSOrZuOHl7Z1sfvQl+s8lZumnj/Wy\n+dGXAFyJflg6TcW9do+vtsxCKdWWqRRGZlcrgBoRZj/zn1yyf//QmNTXe74WkXmjmTu3hjNnf5Dl\nCLKwrKCQu/FK4Cx/Z0K4M2mcDMv+kD6jlyowNmks69gQserOrZw+li3YI8fUccs/zM37/UyhhcQG\nq3J77jvuf8bxxtN0+zWuzhHGJ4Sw2DKVEGNXlrjfGHbNfR/Tu7t9fZqxS0V1dE4ZEvOamtGJzXMp\nLh0gbU9AT++b7Np9J197+Q3WD8xldHUViHCifyCxc3daU/nz7o41ag5mz+jtxD7XOQLETuxzjVuk\nimPV8BqoEUz3QGBCWWpqKepPCCr4McZpIfXUuXMlb1QrBquLVlrJhfqEcM+dMIatW+dlPQGI6eFD\n/Y/w7zKX4wPnZ4/Wzl0oc/mDxkn2M3wEnvi77AqVTucIGSPH1DnO8J3IFMfBs/1IbRUX/unlgYlj\nqaklp8Xn42v2AOEX/YrpeAXeFC4LW/EzPyl3V6tM2tvbsypq5iq54LRoexFv244HUv6g9W5sNy5g\noNtF2YgA6tm44dpF06kZli4XNcOquHbRdMfv5HLmBMUFN0xFatN/j0Jq9zg+CRgiYTOtGMH3oo5O\nuWvxBE2QZYmd6gD9/Z7Xs3a2WsLttGj7Nhc5XidXPRxfaLkJKHBdTKoJsp6NGy6bM4Hrbr5iaEY/\nckwd1918Rc4F2zA5cyxKrd2T60kg6JuZGyompeNFi0Ov2yS6xcvm64XgdZvEfJU7U3Fqa3h4wIBk\nz5AP9fYxbeZtWXV9eqljDTc7xhRI+YOGMfaz+WEjwAxmV6gMqchnctmcCa5tmBAeZ04mpTQxt6vu\nmUohawFBLPxWjOB7UUcniD62pXSkyoVb8fXKx5/p+LFm7NY1MnFaPxjZ283p+uFZ4811tUNuHGth\nd6DmYh4ZWMw25tmeK3TlD6rr4CPfjIzvvlS8Kn1civ/fayxRPr5mj+2DnJubWZALvxUj+F7U0Qmi\nj60fTxWFiq8XZM7Y945rZvu0q/inowNM2rYryzHjtPv2jztfYeO0d6aldVKF21rYtTjVeYx9yQXe\nULh0ALqPO4+33FSxAp+JJV6lzGRL9f+XgtMs3Iq/2JtZkLuOK0bwvehBG0QfWz+eKpzSJf/x4x9T\n+6eLfUkbpYr33nHN/Pryq+mvTvx52Tlm7HbfSk0Nh0ddSPegoRoYIFG3Jpdwh7LkgZNTp+HC8scS\nMG7TJ07i+ru1+4fE3qL/3CC/W7vfV8HPNwsv5WYW5NpGxQi+F7thg9hR68dThVO65GxdXdpiNNin\njYpZU0idsW+fdtWQ2FtYC6+WOGeuH9SOHMXGyZfz4kUTgYTYWzP7IAQ9qxtXIXG03g0//+8wmLFg\nfO50wosfkxm+W3KJa7H+/1JxMwsvdi0gyLWNihF88KaOTrlr4/jxVOGULhl+9uzQa6e0UbFrCqkz\n9tN1DbbHZDpmUtcP7JqNZN4kyoVlDbXSSgV7+ltuSnjuMxduB84l8vcq+GnkEtdi/P9e4OcsPMi2\njhVjy4wqfjRft7NbVvf30/JCehNxu7RRrjWFXKSWX7aammSSyzHjZJ/MHN/wo9VsmTuPXVfMZMvc\neWz40eqccRWDXS/bgj39Tnn8EO6iDZpc4lqM/98L/GyuHmRbx4qa4UcVr58qMtMlw3t6aHluJ5e8\n/nracXZpo1LWFKwZ+5SMGTLkd8w019XathNMvUls+NFqxj5wH/XnzgEw9u2j9DxwHxuAD//54rzx\nucXtzScnTnn8EO6iDZpcKY7L5kyg+uBJBne8Rb0x9IhQNftipvucvzfnssteeDkLL8UaWgo6w69Q\nUhuQfGHuXKYeTt8B6JQ2clo7KGRN4ZMTxvDg5ZMLahhyx7QmGqrS/feZN4n6hx8aEvuhsXPnqH/4\nIdexuWF0TXVB47a03p3w2KeS3EX7WOcxZm/bRdPm55m9bVeoGrcHQa7dr2d2Hqb+hSM0kOiJ3ADU\nv3DEtx2t1nrC4Nn+9HgaqiuiubrO8GNAIYvRXq0pFOqesY7NtVA65u2jtt8d87Z9aYWicaogW0hl\nWStPn+G5f2z8h0pbH6hAcjleOu5/pqwWRrv1BICquprIiz2o4McGt2mjIGv/57tJHLtoLGNtRP/Y\nRc6lFYrhxED2P/hc447YeO7v27bLfn3ghWf55MOfrvjNWE44pTjKbWEMYzkIL1HBV7IIuouXE7tu\n+Txzvrc8La3TM2wYPV+8Ne93C7FZullPKBbH9YHaiwAT6iYoQVBuC2NYy0F4hebwQ8TuLZtZeevn\n+Nbihay89XPs3rI56JDKSq7c9mOdx/jm5Vfz4M1/TeeYsQwidI4Zy/YvLcu7YJurAqcdbtYTisXp\nptHcm5KT7utOpIKUkqtbhv165UY7XoWE3Vs289TK79F/7vzsomZYHfOXfImZ864LMLLykOl9h4TI\nWou9dj59gGpgEHLO2mdv20X96weY88qLiVo9dQ1sv/RKeqZMY8f7rrKNZ/uWHzJl6/1c3HOYt+rH\n8/rc25kz7y/9+T0Henhwzz/yySOpPQgE2k5kfT9MdWXKRbkLjYWxo1U+tONVxNiy+pE0sQfoP9fL\nltWPxELwnbzv9+x+hbe/8bcsPnqEkyMbWdf6aTqapg4dY5nnci1+1r9+gOv3PEtV8vSjeru5fs+z\nbAKwE/z2Ncz5zZ1DVS2bet6i6Td3QuPIktMsqYvTb/ScY2LPEe565eEMscfWvhlkXZkgccrvOwmz\nG8HOdUxQlslyoIIfEk45OFCcxoOkpLIDDjjltt8yVZw6egQBDk24hCNjJzqew2lnbuv+55CMB9kq\nkxgHm7WKjfdkd6ay0iwe5NWtxelVd26l6exWrrvgt2nJ1X5TR41NE5Sg6sqEEadyDL2vddH97OGc\nlSj9rlYZ5icEzeGHhFEXjS1oPCgKzYe7xSm3fcHp82mNLXM+TH/tsJznueb1XyYaibeNTvy3fQ3i\nULvcady5L623u2SvXTSdVwavY/PJv+Fk/ziMEU4NjOPIu++1vbEEVVcmjDiVYzi7vTNvly0/O3FZ\nNxNr4de6mYSlE5YKfkiYt/iz1AxLdwLUDKtj3uLPBhSRPZ6UHbDBbqG0pu8c87ZvGHp/cuTonOf4\nk7c28K2XH0jucD3veLmGnbbvTR5fAAAR30lEQVTHN3LK/kROu2E93iVrdZHqGD6fHx1dySN96+n4\n2G9p+sRf2x7vVD/G77oyYSRXq8F8x/tpvQxjW8dUNKUTEqw8/ZbVj3Dq7aOMumgs8xZ/NnT5e0/K\nDthgt/FqzvYnmLbvfP2fhp6zdDeMcDzHXa/8HxoGM/7R9nXzIXmaneZd9HH+KaKWPlobdtufqPXu\nhDUyszNVZpqlfU3JzUwK6SJ17aLpaTl8KE9dmTDiZJ9EyNuYxE/rZdh9/Cr4IWLmvOtCJ/CZFOpR\n371ls+ubWObGq90Dh3lq1w76z/Xy4jta6KmtQwYHMVXnH0zFDGKkikl1tenWxhSGmV4WVv2ajYPX\n0MUoGjlFa9UztHzky/a/pMMu2TQxb1+TflMog3/eujHEzaVjh1PFyYY/Gp+Ww7fGU22VflarDLuP\nvyTBF5FPA23ATOAaY8yOlM/uAD5PwkjxZWPMk6VcSwkHd0xr4gfr1vHep5/igtNdnBzZyNPvnc8X\nbDZqZVpNTx09wlMrvwfg6saW+tSzZc6Hmf52B83HD/Ps1JmcrmtgZG83s1/dzZ1z/wstLVfBdoeC\nZQgts66hZe9T7mfj+TpT+byw60ShfWUrlVzlGM5c0ui4aGotqJq+waGnAS8XVoMsfeyGknz4IjKT\nhA36B8BtluCLyJXAvwHXABOBXwGXGWOyS9ClEGcfflTYvWUzT/zgnzB953e7Su0wPvKF/5El4itv\n/Rynjh7JOseoseNY8tC/FOT2adr8PJ95+klG2ZRebmxsZNmyZYlZ9+NLsH2mb5wMy/5Q2C+bi7bR\n9texkGowA4nrpt5cPEgDKcWR6c6BhBh7XRQtCJdOWXz4xpjdyYtlfrQIWG2M6QVeEZF9JMT/d6Vc\nTwmeLasfSRN7ANN3zna/QC6raaFNRprrah3r7A81e2m5CR63X/D0vA69U/ljC2tuk5rqgew00ON/\nnWiW8pFvqvD7TLl6yYbZx+9XDr8ZeDrl/RvJMSXiFLJfYNRFY+1n+BeNzen2sRP8O6Y18btkGieT\nxsbGlDeT/a1DPzRDP4jjCmEmqaUSMtNAkOiMpfVzfKfQBdUw++mLJa/gi8ivALuk4V3GmLWlBiAi\nS4AlAFOmTCn1dLGlkMXRUsgl4pnMW/xZ23IR8xZ/lq8X6Pb55IQxHH//B3h186+oGTyfGaytraW1\ntfX8gW4dNnZkpFsONX+K9b9+beh/0wUfvITmvQ+nnLuAdGi+J4wy5P/jTiELqn5vzgqKvIJvjPlQ\nEec9BExOeT8pOWZ3/pXASkjk8Iu4VuwpdXG0EHKJeCa5rKbNDrVxclWk/Kt519LeOGKok1djYyOt\nra1DHb4Adw4bi1SBb7gQek+dbzzedZDxx1fQfG4GL5nxnDp6hFEv/BJqe7LPA+Sd7VtPGLnSQNr+\n0FcKWVAtV/qn3PiV0vkF8GMR+TaJRdsZwDM+XSv2lLMOT6H7BZyspndMayq4DSKkNz53PiiPwway\nbZWZDceB2qpB3j/uVV46mfgHPqrGSewh72zfesLIfPpIRdsf+kouZ08mYffTF0uptsw/Af4JGAes\nF5HnjTE3GGN2icga4EWgH7g1n0NHKZ5y1+HxYr+Amw5XvmJnq7Thgtrz/8BP9tXROKzIf/Ab70mI\n/sLvJhZpM28wbtNODsSximYxuF1QDbufvlhKden8DPiZw2f3AveWcn7FHYXk1cNEoW0QPcVl+uRk\n3/l/4AdOX8h7Luwk25Tm5npJt867PwPDRiQE38m6WSD5qmjqzaBwwu6nLxatpVMBRKUOT6hwkT7p\nG6zit0emAnDFBYd55+jDOcTexV2grxt2/PB8Ht8MnJ/Zl7BYm6uKpnUzsAqsWTeDl7d3Fn29ODDi\n6vGM/sSMoRl99eg6bWKuhIOo1OFJpVyuIkfs3DzVw2DYSOg+Do2TONz8KQ69/RrIUT4w4XVqq7Kr\naxoD/Q0Tqe150+WFM3L9HrhzclXR1JLKxRNmP32xqOBXCFGow2NRTleRIy7cPM3AEuttm0OlThFq\nb9+dKMWcy4GTixLdOSPH1NmKvtM4xLOksqKCrwRAaLp7uXHzWDjsrBVMQuxnzIcXfpxnIdjBulmi\nOydXFU0rd58VSRU89MVNWTl9zfdXNir4IcQu3QHRStnkIkrdvYawSwFZdB1MiP27PwOpBdpmzM9+\nn3lTKNGdA/mraGbeDABM8m3qAm/msXFpoRgnVPBDhl264z8eXoExBjMwMDRW9hSIh0TSVZSWArJJ\n3fR1J8Q9X4G2Ke/NmUYqdobtVEUz82YgVefF3sLK6Vuv7T6LguBXYikEr1HBDxl26Y7B/v6s46Lc\n4LyQ3bqhwkoBOVXKdJGLf7n7A/zuyA8Sgn6ujmu7p3OZ9ZlPTcpTbwYPfXGT7TG5cvpRyPdXaikE\nr1FbZsgoJK0R6hRIDmbOu475S77EqLHjQIRRY8cxf8mXonPzKrIFYj6LZC5HjVfkapMY5RaKYW8t\nGBZ0hh8ynNIdTseGlXy2yyi5irIoskBbPotkORw1+dokRrWFYqWWQvAaFfyQYZfuqKqpScvhQ7hT\nIKGwXfqJjaWz45KlPLW6mdPfz3a+WOQT9Fz2Sq9w0yYxii6dSi2F4DUq+CHDaROV3djMedcFv4HJ\nhtDYLv0kxdJ5PveenqqB9Nx7PkEvV5PyXG0So9pCsVJLIXiNCn4IcUp3ZI6FdSYdSdtlCbjdzZpP\n0LVJefEUUgkzzqjgR5iwzqQjabssAbe5dzeCHtUZdhioxFIIXqOCH2EcZ9JHj/CtxQsDS/FE1nZZ\nJIXk3lXQlSBRwY8wOR09xgSW4oliMbdSKCT3HrXSBVGLV8mNGBOeroKzZ882O3bsCDqMyJCZw3di\n1NhxLHnoX8oUVTxxI4yZG6sgcWO47uYrQimiUYs3zojIs8aY2fmO0xl+hMmcSeNw867UxdIw4SZV\nE7VSxVGLV8mPCn7ESXX0rLz1c7FaLI0aQZQqLiUlo6WVKw8trVBBaOercFPu0gWldLvKdUwUSi0o\n9qjgVxCRr1FT4Vy7aDo1w9L/yflZuqCU2jy5jolCqQXFHk3pVBiRrlFT4ZR7Y1UpKZlcx2j+Prqo\n4CtKGSmnD9+u9r01no9y1PVRyo8KfgQIY70cP4nb71sML2/vZMual+k5k+iVUDeimg/cdHnazcRO\n7DPHnRZ1y1XXRykvKvghJ6z1cvwibr9vMby8vZONj+xmcOC8Dbf3zAAb/vVF4HzKJd8s3U3DFd10\nVVmo4IecsNbL8Yu4/b7F8Lu1+9PEfggDW9a8PCTK+Wbp+Xz2memnl7d3surOrQXdAHSnbrhQwQ85\nQVeeLHd6JejfNwrkWlC1UjwWNbVVQ6KemfYpZFG3mPaLfrVsVIpHBT/kBFl50qmh+sZ/XUnvmdOe\n3QBSbyoigl25D908dh6nVE0qdmURBvrS/3ctZGG2mF23ulM3fKgPP+QEuZnKqaF67+lTacXZdm/Z\n7HiO3Vs2s/LWz/GtxQtZeevnso61biqnjh4BYzCD2SuNunksnVwLp3UjqgF3HvxC9gUUY/HUnbrh\noyTBF5EHROQlEWkXkZ+JyOiUz+4QkX0iskdEbig91HgS5GYqN2mU/nO9PPH95bainynmdjcIu5sK\ngFRV6eYxBy6bM4F3fmBi9gcCgvDQFze5EtvL5kzgupuvGJrRjxxT51gYrZhdwlFuil6plJrS2QDc\nYYzpF5FvAncAfyciVwKLgauAicCvROQyY8xAjnMpDgS1mcptQ3UzOGjrpHGzAOt0UzHG8Ler1xUb\nesXzwc9cQdP00UMLonUjqunrGczK4WeSKbZu9wUUY9NUa2f4KGmGb4x5yhhj/YU9DUxKvl4ErDbG\n9BpjXgH2AdeUci2l/Nilk5ywhDwVNwuwTrl5zdnn57I5E7jlH+Zy68PXU1tXY+/cSaEUsS3kaaCU\n7yj+4uWi7V8CP0m+biZxA7B4IzmWhYgsAZYATJkyxcNwlFLJLL9cP3IkvWfPYgbsH9QyBd7NgnPc\numP5Rb68uBeWyGJ2CWuHr3CRV/BF5FeA3f9jdxlj1iaPuQvoBx4tNABjzEpgJSQaoBT6fcVb7GyY\nqc1Tdm/ZzBPfX267uJo5K3cj5nHrjuUXuZw7uRqyqEc+XuQVfGPMh3J9LiJ/AXwMaDXn/XSHgMkp\nh01Kjikhxs0uV+u/bmblbsVcC76Vjl2+3MLO/64e+XhSUkpHRG4E/ifwQWPM2ZSPfgH8WES+TWLR\ndgbwTCnXUvzH7S7XQmblKublIbMUQiaZ/nf1yMeTUnP43wPqgA0iAvC0MeaLxphdIrIGeJFEqudW\ndeiEn0J2uaqQhw8rX/7QFzfZfp56I1CPfDwpSfCNMe/I8dm9wL2lnF8pL0Hu6lW8w80OWj/KH+ua\nQPjRnbbKENoisTJws4PW6+5bpbRTVMqH1tJRhlDHTGXgprSx1+WPdU0gGqjgK2lobr4ycON/99Ij\nr2sC0UBTOoqilIzWzYkGKviKopSM12sCij9oSkdRlJLRlojRQAVfURRP0Lo54UdTOoqiKDFBBV9R\nFCUmqOAriqLEBBV8RVGUmKCCryiKEhNU8BVFUWKCCr6iKEpMUMFXFEWJCXK+K2HwiMgR4LUSTjEW\nsO/iEX6iHDtEO36NPRiiHDuEK/5LjDHj8h0UKsEvFRHZYYyZHXQcxRDl2CHa8WvswRDl2CGa8WtK\nR1EUJSao4CuKosSEShP8lUEHUAJRjh2iHb/GHgxRjh0iGH9F5fAVRVEUZypthq8oiqI4UBGCLyLf\nEJF2EXleRJ4SkYnJcRGR74rIvuTns4KONRMReUBEXkrG9zMRGZ3y2R3J2PeIyA1BxmmHiHxaRHaJ\nyKCIzM74LNSxA4jIjcn49onI7UHHkw8R+aGIHBaRP6SMjRGRDSKyN/nfC4OM0QkRmSwim0XkxeTf\nzNLkeOjjF5F6EXlGRF5Ixv73yfFLRWR78u/nJyIyLOhY82KMifwPcEHK6y8DDydffxR4AhDgvcD2\noGO1iX0+UJN8/U3gm8nXVwIvAHXApcB+oDroeDNinwlcDvw/YHbKeBRir07GNQ0Yloz3yqDjyhPz\nB4BZwB9Sxv4RuD35+nbr7ydsP0ATMCv5ehTwcvLvJPTxJ/VjZPJ1LbA9qSdrgMXJ8YeBvwk61nw/\nFTHDN8acTHk7ArAWJhYBj5gETwOjRaSp7AHmwBjzlDGmP/n2aWBS8vUiYLUxptcY8wqwD7gmiBid\nMMbsNsbssfko9LGTiGefMeaAMeYcsJpE3KHFGPMb4FjG8CJgVfL1KuDjZQ3KJcaYDmPMc8nXp4Dd\nQDMRiD+pH6eTb2uTPwa4HvhpcjyUsWdSEYIPICL3ishB4Gbg7uRwM3Aw5bA3kmNh5S9JPJFA9GJP\nJQqxRyFGN1xsjOlIvu4ELg4yGDeIyFTgahIz5UjELyLVIvI8cBjYQOLp8ETKZC0Sfz+REXwR+ZWI\n/MHmZxGAMeYuY8xk4FHgS8FGm06+2JPH3AX0k4g/NLiJXQkHJpFbCLXtTkRGAo8BX8l4Mg91/MaY\nAWPMe0g8gV8DXBFwSEURmSbmxpgPuTz0UeCXwNeBQ8DklM8mJcfKSr7YReQvgI8Brck/eohI7A6E\nIvY8RCFGN7wlIk3GmI5kuvJw0AE5ISK1JMT+UWPM48nhyMQPYIw5ISKbgWtJpIhrkrP8SPz9RGaG\nnwsRmZHydhHwUvL1L4DPJt067wW6Uh4fQ4GI3Aj8T+C/GmPOpnz0C2CxiNSJyKXADOCZIGIsgijE\n/p/AjKTTYhiwmETcUeMXwC3J17cAawOMxREREeCfgd3GmG+nfBT6+EVknOWeE5EG4MMk1iA2A59K\nHhbK2LMIetXYix8Ss4Y/AO3AOqDZnF9df4hEvu33pDhJwvJDYkHzIPB88ufhlM/uSsa+B/hI0LHa\nxP4nJHKXvcBbwJNRiT0Z40dJuEX2A3cFHY+LeP8N6AD6kv+7fx64CNgI7AV+BYwJOk6H2N9PIl3T\nnvK3/tEoxA+0ADuTsf8BuDs5Po3ERGYf8H+BuqBjzfejO20VRVFiQkWkdBRFUZT8qOAriqLEBBV8\nRVGUmKCCryiKEhNU8BVFUWKCCr6iKEpMUMFXFEWJCSr4iqIoMeH/A4FEtNuuB85eAAAAAElFTkSu\nQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "OtcekKp375sD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# test_labels[0:111]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jHpRTy2C75m4",
        "colab_type": "code",
        "outputId": "d5909492-066d-4ccb-cc6f-af0ad32b53b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "hidden1_output.shape"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000, 300)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "metadata": {
        "id": "NjlyAKCD_IuQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Code to compute DI score of a layer"
      ]
    },
    {
      "metadata": {
        "id": "-Q2SN_yJ_IGC",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def getDIScoreOfLayer(layer_output, true_labels):\n",
        "  # computeScatterMatrix\n",
        "  rho = 1e-6\n",
        "  mu = layer_output.mean(axis=0)\n",
        "#   print(mu.shape)\n",
        "  mu = mu.reshape((1,-1))\n",
        "  centered_data = layer_output - mu\n",
        "  S_bar = np.matmul(centered_data.T,centered_data)\n",
        "  # compute between class matrix\n",
        "  total_classes = true_labels.max()\n",
        "  S_B = np.zeros(shape = (mu.shape[0],mu.shape[0]))\n",
        "  S_W = np.zeros(shape = (mu.shape[0],mu.shape[0]))\n",
        "\n",
        "  for l in range(0,total_classes):\n",
        "    class_l_index = np.where(true_labels == l)\n",
        "    class_l_output = layer_output[class_l_index[0],:]\n",
        "    mu_l = class_l_output.mean(axis=0)\n",
        "    mu_l = mu_l.reshape((1,-1))\n",
        "    total_examples_this_class = class_l_index[0].shape   \n",
        "    ss = mu_l - mu\n",
        "    S_B = S_B + total_examples_this_class*np.matmul((mu_l - mu).T,(mu_l - mu))\n",
        "    \n",
        "  # compute within class scatter matrix\n",
        "#   for l in range(0,total_classes):\n",
        "#     class_l_index = np.where(true_labels == l)\n",
        "#     class_l_output = layer_output[class_l_index[0],:]\n",
        "#     mu_l = class_l_output.mean(axis=0)\n",
        "#     mu_l = mu_l.reshape((1,-1))\n",
        "#     total_examples_this_class = class_l_index[0].shape\n",
        "#     class_l_index_list = np.ndarray.tolist(class_l_index[0])\n",
        "#     for ex in class_l_index_list:\n",
        "#       S_W = S_W + np.matmul((layer_output[ex,:] - mu_l),(layer_output[ex,:] - mu_l))\n",
        "#   print(S_bar.shape)\n",
        "#   print(S_B.shape)\n",
        "  DI_Score = np.trace(np.matmul(np.linalg.inv(S_bar + rho*np.eye(S_bar.shape[0])),S_B))\n",
        "  return DI_Score\n",
        "    \n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rFfDgZk0M3bp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "97993e60-098e-4416-e271-682e46948b1b"
      },
      "cell_type": "code",
      "source": [
        "cnn1_output.shape"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10000, 32, 32, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 77
        }
      ]
    },
    {
      "metadata": {
        "id": "N4bV-WkGIzTQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def getDIScoreOfLayerCNNLayer(cnn_outputs, true_labels):\n",
        "  # computeScatterMatrix\n",
        "  number_of_filters = cnn_outputs.shape[3]\n",
        "  cnn_di_scores = []\n",
        "  for filt in range(0,number_of_filters):\n",
        "    outputs_filter = cnn_outputs[:,:,:,filt]\n",
        "    outputs_filter_reshaped = np.reshape(outputs_filter, newshape=(cnn_outputs.shape[0],-1))\n",
        "    this_filter_score = getDIScoreOfLayer(outputs_filter_reshaped, true_labels)\n",
        "    cnn_di_scores.append(this_filter_score)\n",
        "  return cnn_di_scores\n",
        "    \n",
        "    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rSVYmUE2Ja8w",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "d = getDIScoreOfLayer(hidden1_output, y_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "P00VY-So_IDI",
        "colab_type": "code",
        "outputId": "4b77e84a-3ce9-47ab-c4d0-fffba1d2679a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "d\n"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7.28717227554946"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "metadata": {
        "id": "HxPPF2n1OE_9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "outputId": "765cbc32-77ba-49c2-d85c-bd1822187482"
      },
      "cell_type": "code",
      "source": [
        "getDIScoreOfLayerCNNLayer(cnn1_output, y_test)"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[6.469722149569672,\n",
              " 5.908691721587186,\n",
              " 6.053436663745952,\n",
              " 6.034774848951665,\n",
              " 5.845621298618829,\n",
              " 6.0649492754093375]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "metadata": {
        "id": "Q74zDFA-OE1W",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "c2qH693z_H5_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "test_eval = classifier.evaluate(X_test, test_Y_one_hot, verbose=0)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wWVD_rPe75gt",
        "colab_type": "code",
        "outputId": "72d32490-a2f5-449d-e025-07e5036222a4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "test_eval"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.05037166946069556, 0.9859]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "metadata": {
        "id": "55PwFf5nb5Em",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}