MNIST_using_dense_layers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MLP for MNIST classification using keras\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.14.0\n",
      "2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "print(keras.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "uint8\n"
     ]
    }
   ],
   "source": [
    "fashion_mnist = keras.datasets.fashion_mnist\n",
    "(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()\n",
    "print(X_train_full.shape)\n",
    "print(X_train_full.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_valid, X_train = X_train_full[:5000]/255.0, X_train_full[5000:]/255.0\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
    "\n",
    "class_names = [\"T-shirt/top\", \"Trouser\", \"Pullover\", \"Dress\", \"Coat\", \"Sandal\", \"Shirt\", \"Sneaker\", \"Bag\", \"Ankle boot\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = keras.models.Sequential()\n",
    "# model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "# model.add(keras.layers.Dense(300, activation='relu'))\n",
    "# model.add(keras.layers.Dense(100, activation='relu'))\n",
    "# model.add(keras.layers.Dense(10, activation='softmax'))\n",
    "\n",
    "model = keras.models.Sequential([keras.layers.Flatten(input_shape=[28, 28]), keras.layers.Dense(300, activation='relu'), keras.layers.Dense(100, activation='relu'), keras.layers.Dense(10, activation='softmax')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tensorflow.python.keras.layers.core.Flatten at 0xb24a478cc0>,\n",
       " <tensorflow.python.keras.layers.core.Dense at 0xb24a481128>,\n",
       " <tensorflow.python.keras.layers.core.Dense at 0xb24a481668>,\n",
       " <tensorflow.python.keras.layers.core.Dense at 0xb24a481940>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights : [[-0.03706561  0.0584991   0.04139393 ... -0.02005192 -0.03405794\n",
      "   0.00971507]\n",
      " [ 0.06406698 -0.02573347  0.00177292 ... -0.06488322  0.04946151\n",
      "   0.01180211]\n",
      " [-0.06617234  0.0386883  -0.06145831 ... -0.05014598 -0.00254269\n",
      "   0.03928738]\n",
      " ...\n",
      " [ 0.06735507 -0.06655463  0.07430497 ...  0.02601893 -0.04017516\n",
      "   0.05063173]\n",
      " [-0.06408674 -0.00879546 -0.00746609 ... -0.04989144  0.00276785\n",
      "  -0.07241197]\n",
      " [-0.0412801  -0.07143273  0.04672633 ... -0.03622402 -0.04020121\n",
      "  -0.0008992 ]]\n",
      "biases : [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "shape : (784, 300), (300,)\n",
      "weights : [[-0.07297421 -0.10983419  0.08999538 ...  0.07757816 -0.07666592\n",
      "  -0.03004466]\n",
      " [-0.07969183  0.08017319  0.08178136 ... -0.0736105   0.02657034\n",
      "   0.08392031]\n",
      " [-0.0085471   0.0837765   0.04300187 ... -0.0015199  -0.00590691\n",
      "  -0.09029882]\n",
      " ...\n",
      " [ 0.04393534 -0.1217728  -0.01296156 ...  0.01066533  0.10197581\n",
      "   0.06044518]\n",
      " [-0.04723365 -0.04641812 -0.05417973 ...  0.07101279 -0.10616542\n",
      "   0.0922023 ]\n",
      " [-0.00741229 -0.09300469  0.01711818 ... -0.05141021  0.05816982\n",
      "  -0.02972297]]\n",
      "biases : [0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0.]\n",
      "shape : (300, 100), (100,)\n",
      "weights : [[-3.08217555e-02  3.58814001e-02  1.94653004e-01 -1.54821754e-01\n",
      "   1.21260524e-01  7.40129650e-02  1.23486757e-01 -1.33057386e-01\n",
      "  -2.22228974e-01  7.92996883e-02]\n",
      " [ 1.53763324e-01 -2.86199600e-02 -5.90573996e-02  6.10208809e-02\n",
      "   5.07926047e-02 -2.01386869e-01  2.16290653e-02 -9.59437639e-02\n",
      "  -3.62680107e-02 -4.10641283e-02]\n",
      " [ 2.29953706e-01 -1.16778515e-01 -1.22916967e-01  4.13982570e-03\n",
      "   1.13227487e-01 -1.61996558e-01  9.11932588e-02 -1.77638575e-01\n",
      "   8.04487467e-02  9.43337679e-02]\n",
      " [ 3.11906636e-03  1.12460881e-01 -4.19138968e-02  7.06803501e-02\n",
      "   1.61965042e-01 -4.71341610e-03  1.25991911e-01 -1.43732116e-01\n",
      "   1.44911021e-01 -6.12193793e-02]\n",
      " [ 1.21191859e-01 -8.16793293e-02  1.32247210e-02 -2.15624794e-01\n",
      "  -1.97479784e-01  5.19925654e-02  2.09793925e-01  6.22065365e-02\n",
      "  -2.06604823e-01  1.81415975e-02]\n",
      " [-2.05683723e-01 -2.80178040e-02  7.41932988e-02 -9.34825987e-02\n",
      "  -3.22057456e-02  2.29838192e-02  1.45760924e-01  6.85373545e-02\n",
      "  -4.29897457e-02  1.52407289e-01]\n",
      " [ 2.22363234e-01  6.92060888e-02 -3.23398411e-03 -1.67892188e-01\n",
      "  -9.42223370e-02  3.06228101e-02 -5.08727729e-02  1.72276020e-01\n",
      "  -1.62977695e-01  5.10142744e-02]\n",
      " [ 2.20268667e-02  1.25153124e-01  1.88913703e-01 -1.38554037e-01\n",
      "   1.18336678e-01  7.85749257e-02  1.10688895e-01 -4.90700603e-02\n",
      "  -2.23967105e-01  1.93390965e-01]\n",
      " [-1.67227089e-02 -3.32086980e-02 -1.26423463e-01 -1.79299563e-02\n",
      "   4.98094559e-02 -2.20369726e-01  9.01524425e-02  1.20048821e-02\n",
      "   1.06393248e-02  6.07240796e-02]\n",
      " [ 1.57301575e-01  1.21137351e-01 -8.72860104e-02 -1.06043309e-01\n",
      "   2.16976941e-01  4.59535122e-02  1.11113638e-01  3.79237831e-02\n",
      "   1.13865137e-01  1.02189183e-03]\n",
      " [-2.29433626e-01 -1.67578399e-01  2.21746981e-01  1.82453841e-01\n",
      "  -8.66090208e-02  3.55484188e-02 -7.99758434e-02 -1.53802752e-01\n",
      "  -9.77796167e-02 -5.51860631e-02]\n",
      " [ 4.34063673e-02  1.67894632e-01  4.95825112e-02  2.15925276e-03\n",
      "  -5.10505736e-02  1.60499305e-01 -1.42415836e-01  7.61273801e-02\n",
      "   6.84494376e-02  2.08769768e-01]\n",
      " [-9.65813249e-02 -5.26481569e-02 -1.18141770e-02 -1.87236637e-01\n",
      "   6.09050095e-02 -7.97981024e-02 -1.67817727e-01 -2.14715883e-01\n",
      "   6.71388209e-02  9.99058485e-02]\n",
      " [-2.14034557e-01 -2.01052040e-01 -1.35300547e-01  2.09773660e-01\n",
      "   1.40264630e-03 -2.79040486e-02  2.22685963e-01 -1.56742066e-01\n",
      "   2.04710573e-01  2.00767666e-01]\n",
      " [ 2.12169677e-01  1.13911539e-01  1.47297859e-01  6.15866780e-02\n",
      "  -1.90336823e-01 -4.61198837e-02 -2.26317406e-01 -1.45071283e-01\n",
      "   7.18834996e-02 -1.77532062e-01]\n",
      " [ 1.39465332e-01 -2.01133847e-01  8.88371766e-02  1.19954795e-02\n",
      "   1.06765568e-01  4.04887497e-02 -1.04939342e-01  4.27954197e-02\n",
      "  -6.59390390e-02  2.06090093e-01]\n",
      " [-9.67156440e-02  6.96848035e-02  1.43587619e-01  1.84373319e-01\n",
      "   7.99728930e-02  5.82547784e-02  2.25065708e-01 -4.07518446e-03\n",
      "   1.21640503e-01 -1.23991862e-01]\n",
      " [-2.73967236e-02  2.27372080e-01 -1.75379589e-01 -1.65343910e-01\n",
      "   9.68418717e-02  1.88657284e-01  7.33824074e-02  1.35686755e-01\n",
      "   2.02066571e-01  1.68528020e-01]\n",
      " [-9.41930413e-02 -9.39437002e-02  2.55972147e-03 -4.59606200e-02\n",
      "  -8.41939002e-02 -1.45899117e-01 -9.64264721e-02  2.18947649e-01\n",
      "  -2.15274885e-01  7.31192529e-02]\n",
      " [ 1.99905664e-01  1.75099015e-01 -1.47251368e-01  9.98598635e-02\n",
      "  -1.50993958e-01 -8.69615525e-02 -2.03000307e-02 -1.71978772e-01\n",
      "  -1.20960779e-01  1.32471710e-01]\n",
      " [ 1.17657959e-01  1.31449103e-01 -8.86795223e-02  7.01096654e-02\n",
      "  -1.86460033e-01 -5.25722057e-02 -2.16428518e-01 -1.96225256e-01\n",
      "  -2.65444964e-02  6.96401596e-02]\n",
      " [-1.89869538e-01  4.86392379e-02 -1.38291761e-01  1.69114769e-02\n",
      "   1.75816268e-01 -1.01278439e-01  1.71732783e-01 -2.05066368e-01\n",
      "   8.31183493e-02 -1.52746677e-01]\n",
      " [ 9.71701741e-02 -1.20807432e-01  9.86821353e-02 -1.33408517e-01\n",
      "   6.47365749e-02  1.67710602e-01  1.99743897e-01  1.07029170e-01\n",
      "  -2.12105542e-01  1.90928519e-01]\n",
      " [-8.96438360e-02  1.61065549e-01 -1.23922676e-02 -1.07085735e-01\n",
      "  -6.96688294e-02 -1.61986053e-03 -6.84763193e-02  1.94394261e-01\n",
      "   2.26458311e-02  1.81250095e-01]\n",
      " [ 2.15177208e-01 -3.59288603e-02  3.53822112e-02  8.94886553e-02\n",
      "   2.09137619e-01  1.50473267e-01  1.91665679e-01  1.15492940e-02\n",
      "  -8.24580044e-02  3.35431397e-02]\n",
      " [-2.05116272e-01  1.04778260e-01 -1.80651173e-01  3.24295461e-03\n",
      "   3.41013968e-02 -1.43996954e-01 -8.45085084e-02 -1.86645627e-01\n",
      "  -8.71662945e-02  6.72354400e-02]\n",
      " [ 1.72584951e-02 -1.61502093e-01 -4.11530435e-02  5.62943816e-02\n",
      "  -9.04369354e-03 -1.09517172e-01 -1.33594826e-01  3.72641683e-02\n",
      "  -1.75565690e-01 -5.62093407e-02]\n",
      " [ 1.81981981e-01 -5.88646829e-02  7.81589150e-02  7.49115050e-02\n",
      "   2.02080280e-01  1.76536053e-01 -1.85565382e-01 -1.58189982e-01\n",
      "  -7.76371062e-02 -1.34349659e-01]\n",
      " [-1.38861120e-01 -1.50103867e-01  2.05022991e-01  1.20845139e-01\n",
      "  -1.21924534e-01  2.72360742e-02  1.85526848e-01 -6.69955015e-02\n",
      "   1.66622609e-01  2.30557084e-01]\n",
      " [-1.04242533e-01 -2.33545512e-01 -2.59410739e-02  4.71516848e-02\n",
      "   2.02100277e-01  6.34786487e-03  9.75922942e-02 -4.51679230e-02\n",
      "  -1.23612277e-01  2.31383681e-01]\n",
      " [-1.43242225e-01  2.25593716e-01 -2.50775367e-02  8.21089745e-02\n",
      "   2.24642366e-01 -1.27775997e-01 -2.13652402e-02  4.28596139e-02\n",
      "   1.28412277e-01  8.42326581e-02]\n",
      " [-1.64593428e-01  5.82482815e-02 -2.68512517e-02  1.28469467e-01\n",
      "  -1.13486119e-01  1.98853135e-01  1.29125863e-01  1.52441055e-01\n",
      "  -1.28203303e-01 -1.22018971e-01]\n",
      " [-9.07411128e-02 -6.93058819e-02  9.78713930e-02  7.15706646e-02\n",
      "  -1.18073247e-01  1.17954522e-01  1.68467999e-01  1.26793981e-04\n",
      "   9.93074477e-02 -4.15709466e-02]\n",
      " [ 1.84291989e-01  2.29325891e-02  1.62450373e-01 -2.06910074e-01\n",
      "  -1.69560105e-01  1.35812432e-01 -3.62852216e-02  1.49644434e-01\n",
      "  -1.82452559e-01 -1.56294852e-02]\n",
      " [ 1.58877969e-02 -1.37111515e-01 -2.70348936e-02 -1.24594457e-01\n",
      "  -2.14514986e-01 -2.18195885e-01 -3.31840962e-02 -1.92244560e-01\n",
      "   1.20699406e-01  7.32347965e-02]\n",
      " [-1.27769589e-01 -3.41451168e-02 -3.39698792e-02  6.17074966e-02\n",
      "  -1.53758600e-01  1.34255946e-01 -1.79558188e-01 -6.44300878e-02\n",
      "   1.71609432e-01  1.15994394e-01]\n",
      " [-1.89401418e-01 -1.28746867e-01 -1.90175742e-01 -1.63838089e-01\n",
      "   2.26163507e-01 -3.72029245e-02  1.87817007e-01  1.46660626e-01\n",
      "  -8.90865624e-02  1.58547401e-01]\n",
      " [-5.94617426e-03  5.31737506e-02 -1.43674150e-01  2.26102144e-01\n",
      "   2.14410961e-01  1.02103084e-01 -1.17293745e-01  9.61930454e-02\n",
      "  -1.82310343e-01 -1.95426941e-02]\n",
      " [ 2.49303281e-02  1.86039507e-02  9.31573510e-02  2.91137397e-02\n",
      "   1.11448735e-01 -1.83763266e-01  1.75968051e-01  4.76794243e-02\n",
      "   2.27449834e-01  7.97727704e-02]\n",
      " [-1.77927732e-01  1.77482098e-01  2.10066497e-01 -6.97060227e-02\n",
      "  -5.65770715e-02 -1.17832869e-01 -1.70866743e-01 -4.25435007e-02\n",
      "   8.53151381e-02 -2.12088287e-01]\n",
      " [ 1.68367326e-02 -2.12052137e-01  4.33652103e-02  1.86753273e-03\n",
      "   1.11632615e-01  1.40430182e-01  4.11005318e-02 -1.87304616e-03\n",
      "   5.68935573e-02  9.16617811e-02]\n",
      " [-1.44442573e-01  1.31885260e-01  9.37589407e-02 -2.94148922e-03\n",
      "   1.10292107e-01 -1.41390115e-01  2.28130311e-01 -2.08535850e-01\n",
      "   6.35832548e-05 -6.53607696e-02]\n",
      " [ 1.68159604e-04 -1.95153475e-01 -1.24998048e-01  5.44164181e-02\n",
      "   1.47821844e-01 -1.18249483e-01 -1.04651406e-01  7.76414573e-02\n",
      "  -1.34097472e-01 -5.92491627e-02]\n",
      " [-6.31467849e-02 -1.85526788e-01  1.01342529e-01  1.06680095e-01\n",
      "   1.23897880e-01  7.86399543e-02  9.74626243e-02  9.43462253e-02\n",
      "  -1.80726200e-02  6.28802776e-02]\n",
      " [ 1.44045442e-01  6.81984127e-02 -5.81827909e-02 -1.85876980e-01\n",
      "   1.66563600e-01 -2.32168540e-01 -2.10288733e-01  9.63862240e-02\n",
      "   4.56675291e-02  1.26766741e-01]\n",
      " [ 1.40633613e-01  2.31925756e-01 -1.64932013e-03 -1.18582077e-01\n",
      "   1.51096821e-01  1.47262841e-01  2.23936647e-01  1.11121625e-01\n",
      "  -1.20594501e-02 -9.26319361e-02]\n",
      " [-8.56833160e-03  1.09788567e-01 -5.32460213e-02  5.04624546e-02\n",
      "   8.00873041e-02 -1.43541798e-01 -1.05750754e-01  2.17767745e-01\n",
      "   5.68246245e-02  1.59585625e-01]\n",
      " [-1.54222667e-01 -1.02370754e-01  1.29819095e-01  1.75479323e-01\n",
      "   1.33776069e-01 -1.05708927e-01 -1.12734437e-02  1.19708091e-01\n",
      "   8.98682475e-02  1.51512325e-01]\n",
      " [ 3.92647982e-02  6.48076832e-02 -1.60997674e-01 -1.00114003e-01\n",
      "  -5.61250448e-02  2.19944865e-01  1.67624354e-01  6.80528581e-02\n",
      "  -1.15157984e-01 -1.25018030e-01]\n",
      " [ 1.97960705e-01  1.46833926e-01  1.78618252e-01  1.65680408e-01\n",
      "  -2.32489854e-02 -1.92595482e-01  9.38202441e-02  1.81823850e-01\n",
      "  -7.59208053e-02  1.13174230e-01]\n",
      " [ 2.82791257e-02  1.56191707e-01  6.82300329e-02  4.53129411e-02\n",
      "  -1.51831150e-01 -2.19642013e-01  1.29646599e-01  2.06328541e-01\n",
      "   1.75337493e-01 -8.26454312e-02]\n",
      " [ 1.59233809e-01  1.41549855e-01 -1.43406153e-01 -1.20654136e-01\n",
      "   1.22577399e-02 -1.61861688e-01 -1.88178450e-01 -1.42273903e-01\n",
      "   6.37815595e-02 -2.77837217e-02]\n",
      " [-1.14335664e-01  5.07768989e-03  2.20855266e-01 -1.60969168e-01\n",
      "  -1.70391321e-01 -7.89431483e-02  2.09558964e-01  1.31080538e-01\n",
      "  -1.76364169e-01 -9.57903564e-02]\n",
      " [-1.93357214e-01 -1.33044511e-01 -6.38101250e-02 -1.58880785e-01\n",
      "  -1.32671893e-01 -1.51703805e-02  1.28086478e-01 -9.74265933e-02\n",
      "  -1.42706111e-01 -1.57364219e-01]\n",
      " [ 1.11311644e-01 -1.50327504e-01 -1.93143219e-01 -1.52241752e-01\n",
      "  -7.17516243e-02 -2.26251259e-01 -7.65949637e-02 -2.30630532e-01\n",
      "   4.67175245e-02  1.62388951e-01]\n",
      " [-7.65836239e-03 -4.71152067e-02 -2.11546049e-01  2.26435572e-01\n",
      "   1.45403534e-01  1.26821816e-01  3.89483571e-02  8.10142756e-02\n",
      "  -8.58458877e-03  1.19749248e-01]\n",
      " [ 1.14229649e-01 -1.28522515e-04 -2.10792929e-01  9.62728560e-02\n",
      "  -1.33653283e-01 -2.94125974e-02  6.10016584e-02 -1.77484721e-01\n",
      "   1.46778613e-01 -8.03942382e-02]\n",
      " [ 1.01046115e-01 -2.18855157e-01 -6.11462146e-02 -4.65626717e-02\n",
      "   2.04317212e-01  2.22475320e-01  9.56066549e-02  1.99257404e-01\n",
      "   1.65036708e-01 -1.31513298e-01]\n",
      " [ 1.56590074e-01  2.18677759e-01 -1.08097821e-01 -1.39685884e-01\n",
      "   1.22758001e-01  2.13596344e-01  1.95832700e-01  9.00587440e-03\n",
      "   1.02056444e-01  1.23659819e-02]\n",
      " [-1.09232634e-01  1.72629148e-01  1.61262602e-01  4.05841470e-02\n",
      "   1.81857646e-02  7.41600692e-02 -2.24458560e-01  1.17023587e-02\n",
      "  -1.55912340e-02 -1.02747232e-02]\n",
      " [ 8.17439258e-02  6.84008002e-04 -1.20225213e-01 -1.61103010e-02\n",
      "   1.59892529e-01 -2.14892954e-01 -6.89958930e-02 -4.30075526e-02\n",
      "  -2.15269759e-01 -1.52861997e-01]\n",
      " [-1.59341842e-01  2.22645819e-01 -4.51839715e-02 -2.07735181e-01\n",
      "   2.94671655e-02 -8.80375654e-02 -1.76398009e-02 -3.01614702e-02\n",
      "   2.01375127e-01 -1.01989612e-01]\n",
      " [ 2.02194214e-01 -1.81846559e-01  1.83498383e-01 -4.04546261e-02\n",
      "   7.38190114e-03 -2.00844184e-01 -1.74338937e-01  5.83307445e-02\n",
      "  -1.92389563e-01 -1.57058522e-01]\n",
      " [ 2.11125374e-01 -1.76277474e-01  2.29594171e-01 -1.61299258e-01\n",
      "  -9.72725749e-02  1.13747895e-01 -1.13113098e-01 -1.89414442e-01\n",
      "   6.86843097e-02  7.87357390e-02]\n",
      " [-2.18209028e-01 -1.57442510e-01  6.12617135e-02 -7.74362683e-02\n",
      "   1.96708202e-01 -7.99898207e-02  9.35204625e-02 -5.00656068e-02\n",
      "  -2.16024816e-01 -8.63827318e-02]\n",
      " [ 5.53452671e-02 -1.68261290e-01  1.05017453e-01 -1.55196831e-01\n",
      "   2.26237506e-01 -2.21642852e-01  1.76594138e-01  2.22044826e-01\n",
      "   1.61055803e-01 -2.01521903e-01]\n",
      " [ 4.30063903e-03  1.22684002e-01  2.19307750e-01 -9.22835320e-02\n",
      "  -2.96720266e-02  1.74167752e-02 -8.25715363e-02  4.87038493e-03\n",
      "   2.01694220e-01  1.93121731e-02]\n",
      " [ 1.52947426e-01 -2.29872078e-01 -3.37794423e-02  7.16550350e-02\n",
      "   2.32054561e-01 -1.40252516e-01  1.94451809e-02  1.91510230e-01\n",
      "  -1.27607718e-01 -5.54316193e-02]\n",
      " [-2.17902377e-01  1.66135460e-01  6.93892837e-02  1.04744464e-01\n",
      "   1.86573356e-01 -8.96612108e-02 -3.66184860e-02 -5.53091168e-02\n",
      "   1.41950876e-01  1.06379837e-02]\n",
      " [-8.09819698e-02  4.23030555e-02 -2.57499665e-02 -1.58565640e-02\n",
      "  -1.88592017e-01  1.77113056e-01  1.27161711e-01  3.75390351e-02\n",
      "   4.02057171e-02  1.77354038e-01]\n",
      " [ 3.08921933e-03  1.19173586e-01 -1.59546018e-01  1.43940151e-01\n",
      "   1.55267566e-02  7.38245249e-02 -1.58376306e-01 -3.09312344e-02\n",
      "  -1.52782381e-01  1.59870893e-01]\n",
      " [-2.45046765e-02  2.41526961e-02 -9.43268836e-03  4.38564420e-02\n",
      "  -7.18539804e-02 -1.81739151e-01  1.13225132e-01 -1.90770596e-01\n",
      "   2.88513303e-02  1.62199706e-01]\n",
      " [-9.70241129e-02  3.34670842e-02  4.60438132e-02  4.13400233e-02\n",
      "  -2.11557597e-02  1.81874901e-01  1.25715405e-01 -2.13135108e-01\n",
      "   3.02927792e-02 -1.90457210e-01]\n",
      " [-1.15516581e-01 -3.81323248e-02 -9.55938548e-02 -1.67807996e-01\n",
      "  -1.43892035e-01 -2.02568501e-01  4.34842706e-03 -2.10158318e-01\n",
      "   1.32490754e-01 -2.14407235e-01]\n",
      " [ 1.42400175e-01  2.08645940e-01 -2.15005219e-01  1.57324910e-01\n",
      "   1.87202692e-02  2.01610923e-01 -2.11950973e-01 -2.03651533e-01\n",
      "   5.40746450e-02 -3.48408669e-02]\n",
      " [-1.81079090e-01 -1.55206412e-01 -1.76566094e-02  1.67798519e-01\n",
      "  -2.24312052e-01  1.55136287e-01 -2.91106254e-02  1.43779010e-01\n",
      "  -1.71246827e-02  2.12062895e-01]\n",
      " [-6.75258189e-02  8.11457634e-04 -6.81324899e-02  1.56820148e-01\n",
      "  -2.06465781e-01 -1.25573084e-01 -3.24108899e-02 -6.74578398e-02\n",
      "  -6.98740631e-02 -1.50346309e-02]\n",
      " [ 6.82363808e-02 -1.95450932e-01  9.89753902e-02  1.68918073e-02\n",
      "  -6.74661994e-03  1.93440080e-01 -4.05029953e-03 -1.65861696e-02\n",
      "   5.37439287e-02 -1.00251541e-01]\n",
      " [-6.58798963e-02  1.42965823e-01  1.64018631e-01  6.01827800e-02\n",
      "   2.18543798e-01  2.57461369e-02 -1.11814193e-01 -8.82064998e-02\n",
      "   2.12016404e-02  1.28055841e-01]\n",
      " [ 2.70035565e-02 -2.21969545e-01 -2.02261582e-01 -2.01879099e-01\n",
      "  -1.05771735e-01 -7.16178864e-02 -1.67983711e-01 -1.37820542e-02\n",
      "   1.24865174e-01  7.97484815e-02]\n",
      " [-6.02774620e-02 -1.33888423e-03  2.14836776e-01  2.20669687e-01\n",
      "   1.12356097e-01 -1.97241962e-01  1.28831625e-01 -5.95470667e-02\n",
      "   4.94377315e-03  2.23716855e-01]\n",
      " [-2.09920108e-01  9.88838673e-02 -1.02317974e-01  8.01319182e-02\n",
      "   8.03013146e-02 -2.79120058e-02  1.78752333e-01  3.69336903e-02\n",
      "   2.43849158e-02 -2.17737556e-01]\n",
      " [ 4.39730287e-03 -1.40863404e-01  2.04882056e-01 -1.37215853e-01\n",
      "   4.25291359e-02  4.52232361e-03 -3.32293659e-02 -1.88685283e-01\n",
      "  -1.56920761e-01  9.67664123e-02]\n",
      " [ 2.27651626e-01 -1.73484102e-01  1.65507972e-01 -1.22890517e-01\n",
      "   1.89266175e-01 -4.29786593e-02  1.98172212e-01 -8.06155205e-02\n",
      "  -6.21584058e-03  2.01961517e-01]\n",
      " [-6.48523867e-02  2.71532834e-02 -2.19099775e-01  1.65341139e-01\n",
      "  -1.82908922e-02 -1.40403479e-01 -7.77475387e-02  1.68888718e-01\n",
      "  -1.34283125e-01 -2.10205883e-01]\n",
      " [ 1.27469182e-01  9.17528570e-02  1.12077117e-01  1.10496074e-01\n",
      "  -1.09230742e-01 -4.88166958e-02 -3.34211290e-02 -2.39267945e-04\n",
      "  -8.40320885e-02 -2.71303952e-02]\n",
      " [-9.16788727e-02 -2.28018850e-01  1.28439218e-01 -3.10114622e-02\n",
      "   1.55374855e-01  1.39077336e-01  3.11865807e-02  4.99771833e-02\n",
      "  -2.06339166e-01 -1.82978541e-01]\n",
      " [ 2.74595320e-02  1.62281990e-01  1.43114775e-02  1.17868960e-01\n",
      "   5.05563319e-02 -1.90383196e-02 -1.74834624e-01  2.45359540e-03\n",
      "  -5.78678399e-02 -1.65800348e-01]\n",
      " [-2.02772141e-01 -1.47685751e-01 -2.47709602e-02  1.03094906e-01\n",
      "   3.93730402e-02  1.09028667e-02  1.03270918e-01 -2.26016387e-01\n",
      "   4.17078137e-02  3.32885981e-02]\n",
      " [-4.84788716e-02  2.22620130e-01  2.10612983e-01 -1.27440780e-01\n",
      "   1.50185436e-01 -1.92777395e-01  1.05408311e-01  1.77552313e-01\n",
      "   1.90435380e-01  2.16763735e-01]\n",
      " [-1.31613016e-02  6.72616661e-02  3.44954133e-02  1.53710097e-02\n",
      "   6.66864514e-02 -1.39428407e-01 -1.38189256e-01 -1.37476504e-01\n",
      "   2.06729442e-01  1.47941887e-01]\n",
      " [-1.47888541e-01  7.79477656e-02  1.60501659e-01 -6.65112883e-02\n",
      "   1.20449662e-01  2.13534355e-01 -1.95784420e-01  5.60445786e-02\n",
      "  -8.95390362e-02 -1.68582976e-01]\n",
      " [ 1.98457062e-01  1.36729866e-01  8.18399191e-02  2.79168487e-02\n",
      "  -1.85324386e-01  1.57497674e-01 -1.22496113e-01 -8.02863836e-02\n",
      "   2.03455091e-01  1.79817051e-01]\n",
      " [-9.14665461e-02  2.32066870e-01 -9.44934636e-02  1.52713954e-02\n",
      "   5.87446690e-02  5.26526272e-02 -6.96318001e-02 -1.61780849e-01\n",
      "   4.83089387e-02  3.20219994e-02]\n",
      " [-1.10527977e-01 -2.15023026e-01  1.50530785e-01  9.81656611e-02\n",
      "   2.33110279e-01 -7.44283497e-02 -1.80424720e-01  3.99217308e-02\n",
      "  -1.88111573e-01 -1.20126881e-01]\n",
      " [-1.62991941e-01 -7.02936351e-02  3.88160944e-02  1.15162373e-01\n",
      "   6.81568086e-02 -1.27011240e-01 -6.06790334e-02 -7.86654055e-02\n",
      "  -1.54706150e-01  1.01082325e-02]\n",
      " [ 2.19271779e-02  5.04532754e-02 -1.89853281e-01  1.44074947e-01\n",
      "  -8.57227296e-02 -6.75013810e-02  1.93992376e-01  1.95837766e-01\n",
      "  -1.04492709e-01  5.01805544e-03]\n",
      " [-2.12560296e-01  1.52788341e-01 -2.23412961e-02 -2.07806617e-01\n",
      "   5.71979284e-02  8.94708931e-02 -9.96033847e-02 -1.20543048e-01\n",
      "   1.74276471e-01  1.01530612e-01]\n",
      " [-2.45419890e-02  1.29191875e-02  2.41582692e-02  2.10475832e-01\n",
      "   2.03024268e-01 -1.40766025e-01  7.47789443e-03  1.63468748e-01\n",
      "  -1.21489935e-01 -2.05113366e-01]\n",
      " [ 8.50456953e-02  1.61229074e-03 -1.32921040e-02 -2.03054950e-01\n",
      "  -2.10550889e-01 -8.02074373e-03  1.16906255e-01  4.36699092e-02\n",
      "   2.12844938e-01  1.06978208e-01]]\n",
      "biases : [0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "shape : (100, 10), (10,)\n"
     ]
    }
   ],
   "source": [
    "w1, b1 = model.layers[1].get_weights()\n",
    "w2, b2 = model.layers[2].get_weights()\n",
    "w3, b3 = model.layers[3].get_weights()\n",
    "\n",
    "print(f\"weights : {w1}\\nbiases : {b1}\\nshape : {w1.shape}, {b1.shape}\")\n",
    "print(f\"weights : {w2}\\nbiases : {b2}\\nshape : {w2.shape}, {b2.shape}\")\n",
    "print(f\"weights : {w3}\\nbiases : {b3}\\nshape : {w3.shape}, {b3.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"sparse_categorical_crossentropy\",\n",
    "             optimizer=\"sgd\",\n",
    "             metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 5000 samples\n",
      "Epoch 1/30\n",
      "55000/55000 [==============================] - 11s 192us/sample - loss: 0.7247 - acc: 0.7628 - val_loss: 0.5139 - val_acc: 0.8224\n",
      "Epoch 2/30\n",
      "55000/55000 [==============================] - 13s 242us/sample - loss: 0.4896 - acc: 0.8300 - val_loss: 0.4517 - val_acc: 0.8430\n",
      "Epoch 3/30\n",
      "55000/55000 [==============================] - 11s 208us/sample - loss: 0.4439 - acc: 0.8449 - val_loss: 0.4233 - val_acc: 0.8504\n",
      "Epoch 4/30\n",
      "55000/55000 [==============================] - 12s 214us/sample - loss: 0.4161 - acc: 0.8532 - val_loss: 0.3935 - val_acc: 0.8626\n",
      "Epoch 5/30\n",
      "55000/55000 [==============================] - 11s 207us/sample - loss: 0.3959 - acc: 0.8617 - val_loss: 0.3882 - val_acc: 0.8644\n",
      "Epoch 6/30\n",
      "55000/55000 [==============================] - 13s 229us/sample - loss: 0.3799 - acc: 0.8659 - val_loss: 0.3852 - val_acc: 0.8664\n",
      "Epoch 7/30\n",
      "55000/55000 [==============================] - 9s 165us/sample - loss: 0.3671 - acc: 0.8698 - val_loss: 0.3765 - val_acc: 0.8698\n",
      "Epoch 8/30\n",
      "55000/55000 [==============================] - 12s 226us/sample - loss: 0.3545 - acc: 0.8751 - val_loss: 0.3644 - val_acc: 0.8740\n",
      "Epoch 9/30\n",
      "55000/55000 [==============================] - 9s 170us/sample - loss: 0.3431 - acc: 0.8784 - val_loss: 0.3391 - val_acc: 0.8804\n",
      "Epoch 10/30\n",
      "55000/55000 [==============================] - 12s 226us/sample - loss: 0.3339 - acc: 0.8811 - val_loss: 0.3584 - val_acc: 0.8748\n",
      "Epoch 11/30\n",
      "32192/55000 [================>.............] - ETA: 5s - loss: 0.3266 - acc: 0.8851"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=30, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "pd.DataFrame(history.history).plot(figsize=(8, 5))\n",
    "plt.grid(True)\n",
    "plt.gca().set_ylim(0, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.evaluate(X_test/255.0, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}