dnn_methods.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/yanjia/anaconda3/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import DataFrame\n",
    "from pandas import read_csv\n",
    "from matplotlib import pyplot\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.layers import Flatten\n",
    "from tensorflow.keras.layers import Dropout\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "#from keras_layer_normalization import LayerNormalization\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import MaxPooling1D\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import LearningRateScheduler\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.regularizers import l1\n",
    "#from tensorflow.keras.layers import LayerNormalization\n",
    "\n",
    "# supress deprecation warnings from TF\n",
    "import tensorflow as tf\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set=read_csv('data/train_set.csv')\n",
    "val_set=read_csv('data/val_set.csv')\n",
    "test_set=read_csv('data/test_set.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainy=read_csv('data/trainy.csv')\n",
    "valy=read_csv('data/valy.csv')\n",
    "testy=read_csv('data/testy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_csv('data/orginal_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make the range between 0 and 1\n",
    "column = list(train_set.columns.values)\n",
    "for col in column:\n",
    "    minimum = df[col].min()\n",
    "    maximum = df[col].max()\n",
    "    train_set[col] = (train_set[col] - minimum)/(maximum-minimum)\n",
    "    val_set[col] = (val_set[col] - minimum)/(maximum-minimum)\n",
    "    test_set[col] = (test_set[col] - minimum)/(maximum-minimum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_cnn = train_set.values\n",
    "val_set_cnn = val_set.values\n",
    "test_set_cnn = test_set.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_cnn = np.reshape(train_set_cnn, (train_set_cnn.shape[0],train_set_cnn.shape[1],1))\n",
    "val_set_cnn = np.reshape(val_set_cnn, (val_set_cnn.shape[0],val_set_cnn.shape[1],1))\n",
    "test_set_cnn = np.reshape(test_set_cnn, (test_set_cnn.shape[0],test_set_cnn.shape[1],1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_4 (Conv1D)            (None, 25, 64)            128       \n",
      "_________________________________________________________________\n",
      "conv1d_5 (Conv1D)            (None, 25, 128)           8320      \n",
      "_________________________________________________________________\n",
      "conv1d_6 (Conv1D)            (None, 25, 256)           33024     \n",
      "_________________________________________________________________\n",
      "conv1d_7 (Conv1D)            (None, 25, 256)           65792     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 25, 256)           0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 6400)              0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               819328    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 926,721\n",
      "Trainable params: 926,721\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "keras_model_weights = \"final_model/keras2_conv1d.h5\"\n",
    "keras_model_json = \"final_model/keras2_conv1d.json\"\n",
    "\n",
    "keras_model = model_from_json(open(keras_model_json).read())\n",
    "keras_model = tensorflow.keras.models.load_model(keras_model_weights)\n",
    "keras_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 175976 samples, validate on 24531 samples\n",
      "Epoch 1/50\n",
      "175976/175976 [==============================] - 3s 16us/sample - loss: 0.8956 - acc: 0.8152 - val_loss: 0.6385 - val_acc: 0.7941\n",
      "Epoch 2/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7679 - acc: 0.8410 - val_loss: 0.4952 - val_acc: 0.8969\n",
      "Epoch 3/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7474 - acc: 0.8536 - val_loss: 0.4688 - val_acc: 0.9143\n",
      "Epoch 4/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7414 - acc: 0.8569 - val_loss: 0.5609 - val_acc: 0.8568\n",
      "Epoch 5/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7373 - acc: 0.8600 - val_loss: 0.5748 - val_acc: 0.8568\n",
      "Epoch 6/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7322 - acc: 0.8637 - val_loss: 0.5412 - val_acc: 0.8682\n",
      "Epoch 7/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7325 - acc: 0.8639 - val_loss: 0.5666 - val_acc: 0.8588\n",
      "Epoch 8/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7309 - acc: 0.8658 - val_loss: 0.4776 - val_acc: 0.9118\n",
      "Epoch 9/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7299 - acc: 0.8668 - val_loss: 0.6098 - val_acc: 0.8351\n",
      "Epoch 10/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7276 - acc: 0.8679 - val_loss: 0.5884 - val_acc: 0.8489\n",
      "Epoch 11/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.7239 - acc: 0.8692 - val_loss: 0.6467 - val_acc: 0.8215\n",
      "Epoch 12/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5452 - acc: 0.8799 - val_loss: 0.4504 - val_acc: 0.8442\n",
      "Epoch 13/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5421 - acc: 0.8804 - val_loss: 0.3961 - val_acc: 0.8672\n",
      "Epoch 14/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5404 - acc: 0.8818 - val_loss: 0.3714 - val_acc: 0.8783\n",
      "Epoch 15/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5391 - acc: 0.8828 - val_loss: 0.3837 - val_acc: 0.8726\n",
      "Epoch 16/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5383 - acc: 0.8830 - val_loss: 0.3774 - val_acc: 0.8763\n",
      "Epoch 17/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5373 - acc: 0.8830 - val_loss: 0.3827 - val_acc: 0.8716\n",
      "Epoch 18/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5368 - acc: 0.8834 - val_loss: 0.3861 - val_acc: 0.8704\n",
      "Epoch 19/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5359 - acc: 0.8836 - val_loss: 0.3936 - val_acc: 0.8693\n",
      "Epoch 20/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5350 - acc: 0.8840 - val_loss: 0.3817 - val_acc: 0.8720\n",
      "Epoch 21/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5352 - acc: 0.8838 - val_loss: 0.4190 - val_acc: 0.8583\n",
      "Epoch 22/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5153 - acc: 0.8858 - val_loss: 0.3698 - val_acc: 0.8709\n",
      "Epoch 23/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5148 - acc: 0.8859 - val_loss: 0.3771 - val_acc: 0.8688\n",
      "Epoch 24/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5146 - acc: 0.8863 - val_loss: 0.3663 - val_acc: 0.8728\n",
      "Epoch 25/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5145 - acc: 0.8865 - val_loss: 0.3749 - val_acc: 0.8707\n",
      "Epoch 26/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5144 - acc: 0.8866 - val_loss: 0.3840 - val_acc: 0.8671\n",
      "Epoch 27/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5143 - acc: 0.8867 - val_loss: 0.3668 - val_acc: 0.8729\n",
      "Epoch 28/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5142 - acc: 0.8868 - val_loss: 0.3683 - val_acc: 0.8720\n",
      "Epoch 29/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5141 - acc: 0.8869 - val_loss: 0.3654 - val_acc: 0.8736\n",
      "Epoch 30/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5142 - acc: 0.8867 - val_loss: 0.3686 - val_acc: 0.8713\n",
      "Epoch 31/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5139 - acc: 0.8873 - val_loss: 0.3814 - val_acc: 0.8683\n",
      "Epoch 32/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5139 - acc: 0.8870 - val_loss: 0.3693 - val_acc: 0.8721\n",
      "Epoch 33/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5138 - acc: 0.8870 - val_loss: 0.3710 - val_acc: 0.8714\n",
      "Epoch 34/50\n",
      "175976/175976 [==============================] - 2s 14us/sample - loss: 0.5137 - acc: 0.8874 - val_loss: 0.3685 - val_acc: 0.8726\n",
      "Epoch 35/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5135 - acc: 0.8873 - val_loss: 0.3683 - val_acc: 0.8722\n",
      "Epoch 36/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5135 - acc: 0.8872 - val_loss: 0.3705 - val_acc: 0.8720\n",
      "Epoch 37/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5135 - acc: 0.8876 - val_loss: 0.3746 - val_acc: 0.8700\n",
      "Epoch 38/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5133 - acc: 0.8877 - val_loss: 0.3832 - val_acc: 0.8669\n",
      "Epoch 39/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5132 - acc: 0.8873 - val_loss: 0.3679 - val_acc: 0.8731\n",
      "Epoch 40/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5133 - acc: 0.8878 - val_loss: 0.3713 - val_acc: 0.8716\n",
      "Epoch 41/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5132 - acc: 0.8875 - val_loss: 0.3742 - val_acc: 0.8711\n",
      "Epoch 42/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5111 - acc: 0.8877 - val_loss: 0.3712 - val_acc: 0.8712\n",
      "Epoch 43/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8877 - val_loss: 0.3716 - val_acc: 0.8711\n",
      "Epoch 44/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8878 - val_loss: 0.3694 - val_acc: 0.8715\n",
      "Epoch 45/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8879 - val_loss: 0.3694 - val_acc: 0.8716\n",
      "Epoch 46/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8880 - val_loss: 0.3731 - val_acc: 0.8708\n",
      "Epoch 47/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8879 - val_loss: 0.3727 - val_acc: 0.8703\n",
      "Epoch 48/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8876 - val_loss: 0.3699 - val_acc: 0.8713\n",
      "Epoch 49/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8878 - val_loss: 0.3720 - val_acc: 0.8708\n",
      "Epoch 50/50\n",
      "175976/175976 [==============================] - 3s 15us/sample - loss: 0.5110 - acc: 0.8878 - val_loss: 0.3695 - val_acc: 0.8716\n"
     ]
    }
   ],
   "source": [
    "##### DNN model input_shape = (25,)\n",
    "dnn_model = Sequential()\n",
    "dnn_model.add(Dense(64, input_shape=(train_set.shape[1],), kernel_regularizer=l1(0.001), activation='relu'))\n",
    "dnn_model.add(Dense(128, kernel_regularizer=l1(0.001), activation='relu'))\n",
    "dnn_model.add(Dense(256, kernel_regularizer=l1(0.001), activation='relu'))\n",
    "dnn_model.add(Dense(256, kernel_regularizer=l1(0.001), activation='relu'))\n",
    "dnn_model.add(Dense(128, activation='relu', kernel_regularizer=l1(0.001)))\n",
    "\n",
    "dnn_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "def schedule(epoch):\n",
    "    if epoch<=10:\n",
    "        return 1e-2\n",
    "    elif epoch<= 20:\n",
    "        return 1e-3\n",
    "    elif epoch<=40:\n",
    "        return 1e-4\n",
    "    else:\n",
    "        return 1e-5\n",
    "\n",
    "lr_schedule= LearningRateScheduler(schedule)\n",
    "class_weight = {0: 3.,\n",
    "                1: 1.}\n",
    "\n",
    "dnn_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "history = dnn_model.fit(train_set, trainy, validation_data=(val_set, valy), epochs=50, batch_size=128, \n",
    "                    shuffle=True, callbacks=[lr_schedule], class_weight=class_weight, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_34 (Dense)             (None, 64)                1664      \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dense_36 (Dense)             (None, 256)               33024     \n",
      "_________________________________________________________________\n",
      "dense_37 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 141,825\n",
      "Trainable params: 141,825\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19584/19584 [==============================] - 0s 6us/sample - loss: 0.3616 - acc: 0.8707\n"
     ]
    }
   ],
   "source": [
    "# evaluate model\n",
    "_, accuracy_test1 = dnn_model.evaluate(test_set, testy, batch_size=128, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24531/24531 [==============================] - 0s 11us/sample - loss: 0.3695 - acc: 0.8716\n"
     ]
    }
   ],
   "source": [
    "_, accuracy_test1 = dnn_model.evaluate(val_set, valy, batch_size=128, verbose=1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhU5dn48e89k5UAYY9AEJBFNgErILgGERWquJS61KW2Lj+1VFtb31fFBRWXute3+ipvF611wx0tImoJKorsKATZQcIegpB9mbl/f5yTEMIEJsuZSTL357rmmrM855z7GcLcc85zzvOIqmKMMSZ2+aIdgDHGmOiyRGCMMTHOEoExxsQ4SwTGGBPjLBEYY0yMs0RgjDExzhKBMcbEOEsEptkRkU0iUiQi+SKyQ0ReFJGW1cqcJCL/EZE8EdknIh+IyIBqZVqLyNMi8oO7r3XufIfI1sgYb1kiMM3VearaEhgKHA/cUbFCREYBs4H3gS5AT2A5ME9EjnHLJACfAQOBc4DWwEnAHmCEV0GLSJxX+zamJpYITLOmqjuAj3ESQoVHgX+q6p9VNU9Vc1X1LmA+MMUtcxVwNHChqmapalBVd6nqA6o6M9SxRGSgiHwiIrkislNE7nSXvygiU6uUyxCR7Crzm0Tkv0XkW6BARO4Skbeq7fvPIvKMO50qIn8Tke0islVEpoqIv54flYlhlghMsyYi6cA4YJ073wLnl/2bIYpPB8a602cCs1Q1P8zjtAI+BWbhnGX0xjmjCNdlwE+BNsDLwHgRae3u2w9cDLzqln0JKHePcTxwFnBtLY5lzEEsEZjm6j0RyQO2ALuAe93l7XD+7reH2GY7UHH9v30NZWpyLrBDVZ9Q1WL3TOObWmz/jKpuUdUiVd0MLAEucNedARSq6nwRScNJbL9T1QJV3QU8BVxai2MZcxBLBKa5ukBVWwEZQD8OfMHvBYJA5xDbdAZy3Ok9NZSpSTdgfZ0idWypNv8qzlkCwC84cDbQHYgHtovIjyLyI/AC0KkexzYxzhKBadZUdS7wIvC4O18AfA38PETxizlwOedT4GwRSQnzUFuAXjWsKwBaVJk/KlSo1ebfBDLcS1sXciARbAFKgA6q2sZ9tVbVgWHGacwhLBGYWPA0MFZEKhqMbwd+KSI3i0grEWnrNuaOAu5zy7yM86X7toj0ExGfiLQXkTtFZHyIY3wIHCUivxORRHe/J7rrluFc828nIkcBvztSwKq6G8gE/gFsVNVV7vLtOHc8PeHe3uoTkV4icnodPhdjAEsEJga4X6r/BO52578EzgYuwmkH2IzT6HqKqq51y5TgNBh/D3wC7AcW4FxiOuTav6rm4TQ0nwfsANYCo93VL+PcnroJ50v8jTBDf9WN4dVqy68CEoAsnEtdb1G7y1jGHERsYBpjjIltdkZgjDExzhKBMcbEOEsExhgT4ywRGGNMjGtyHVx16NBBe/ToUadtCwoKSEkJ97bw5sHqHBuszrGhPnVevHhxjqp2DLWuySWCHj16sGjRojptm5mZSUZGRsMG1MhZnWOD1Tk21KfOIrK5pnV2acgYY2KcJQJjjIlxlgiMMSbGWSIwxpgYZ4nAGGNinGeJQET+LiK7RGRFDetFRJ5xBwT/VkR+4lUsxhhjaublGcGLOIN+12Qc0Md9XQ/8r4exGGOMqYFnzxGo6uci0uMwRc7HGUBcgfki0kZEOrv9rRtjmhhVRdUZYSeoigbK0dICtLQAVFFVAsEg5QGlLBCgrDxAIKgUlpQhAqBU7EBVUZxtpMp0xXHQIDnbNrByaQIKaNDd1i2nABqsXFaxTwi627vLgwEgCEEoKCklKc7nzgfd9UEUdffl7k+DqAaRyngVCEBlDEE3bnebyu3dmHD2EwgEKC0vJ9Hvq9y3HnScgPvBHli2v0VfnEH3GlY0HyjrysHD82W7yw5JBCJyPc5ZA2lpaWRmZtbpgPn5+XXetqmyOjc/qkpROewpVvYUBdlXouQWlPDeutkUlCkFZZAU53yXBXG+nwLul3RAYVehkpoglAW1cj7ZHyQYDLK3OEiyP0g8AVpQTBvySZBSErWUzrIHFLrJToL4OEa2UaSJJEopbcinhZRwFLm0kBJaUEyKlHj6OQwEWOPpIRqVoArvtf2VJ3/b0UwEEmJZyMERVHUaMA1g2LBhWtcn6+xJxNjQ1OpcHgiyK6+E7fuKyckvYVdeCSu37iM7t5Bd+wtJ8kNJeTklJWXkl5RSUlqOEMRPEB9KHAE6Sy5tJI+2UkAXCujt30HQn0yylNCKIvrrWgIST6/gJkqJJ1gq7vZB/OJmDICkWsYu8eQnplHmS6Y4rjXl/vYUB0vZ3qI7RQntKfcnEx8opDCxIyJCUKEsoLROjsfv8+H3+SgPQkK8H79PAHG+GcSHuF8RIoJzylDx7izbvPkHuvfoASJOWXG3EEHdZeJuJxXLRAAf4gPFD+JzthMhoBDnj8Pnq1jmd959AuJ34/BVvh/Y1le5XHw+d/8H9ovP78Ti87ll3W18PgQf8XFxOLN+fD4fPp+zveBz5/2IT/CJ0M6jv+1oJoJsnAG/K6QD26IUizH1EwxCsAwCpVD0I+zfSsnudezZsoadBUpxSSloOcUlJZSVlVFalE/rshxal+2mDD/Hyha6ksQxlNFW8ms+jo/wv6wVSGwDCS2hZRqUl0DaxSSUF0HbnuCr+CL0u9N+8LlfcIhzaaJNd+eyRKvO4E+AFu0hqTUktob4JOKANiEO3S3Esoa2KzOTAU0o4Tdm0UwEM4BJIvI6cCKwz9oHTMSpQnmx8yorhh83w7rPIFgOZYVQkud8uQfKIFiOBssoLynGt3WhM6+KT8vxhTiZTQS6uK8KZeqnHOeXpo8gJXEtwBfH/nYjaBEsJNBpAPsSk2hJEf7UrlV+ebqvyi/vg19r1qyh78izoXW682Wd2BLiEiP1KZomzrNEICKv4bRqdBCRbOBeIB5AVZ8HZgLjgXVAIfArr2IxhmAA9m+Fbctg21LYuRIK98CetVC8L+QmRb4UCkiiWOMJEEc5PkqCPkrVT672JYCPddqVMvz4fHGktW0F/njK41LwJbclMaU1cZ0Hcewx3enUpjWtkxOI9/mc/wSuiq/q1vWs3raCTPr2yqjnXkys8vKuocuOsF6B33h1fNPMFObCju9g5wpnuvJOioPv5kCD9MneAvkznEsbG+bC3o0hd1lCAhvoyvzgKWRrBwqDcSjC54HBbKM9R7dLoUPLBADatEigXUoCSfE+urZpQctEP33TWnHZUa1ITY53rz0b0zQ1uW6oTQzI2wFZM2DF27BrFQRKnEs3FSqua1e7PKIiBBHalgXJ2+mnLAgl6mdHsDcbtDNLg73J0VSytDvbfWmc1KsjR7VOonVyHAl+H63j/Azv0Zb7e7QjIc4eujexwxKBiQ5V8vfuRDbMIW7TXKQwB3/O9/gKdiGBA7cdBpLbUdL1dEo6j6C0fT+KOxxHILk9JeVBfsgtZN2ufL5an8OG3QVs31d80CES4nykt02mR/sUenVM4ey+HenZIYWjWicR57cvemMqWCIwniopD7BgYy6bduexftUyTij8go77VzK4dBktq91nvjbYlQK6siTYl48CI1iqvSkvjoO9QFZFqUN7LOnYKpEubZK58PiudG2bzL7sdfx6QgZJ8X7P62dMc2CJwNRZMKiUBoKUBYIUlQXYklvIvLU57Nu+nsKCPHZmb2Aoqxgq6/m573uSpKxy282thrCj/Uj2tR1ITpuhlCemOrePi5AocJEIPwN8Iji3kQs+9zZyn3s9vkubZHp1bEm7lISD4sos2mhJwJhasERgwpJbUMp/vt/FrBU7+CG3gA2785FgOUmUkir5DJYNjPEv5Wb/Fwc2qvLXld//ErTHCchRx0H6CLr74+ge+WoYY0KwRGAO64cdu5n38RvkrluET8s52RfkloTNDEj8Hr+WH1K+vEUavuN+hu/oERCXDCkdoMvxtPTZL3RjGitLBDFGVdm+r5iV2/azJ7+EgCrBoFIeVAJBJajO9K4N3/HTLU8yXL/laAA/BHwJ+PzxSGJLaDcCepwCSanOE6etjoIepxDXol20q2iMqSVLBM3c/hLlT7O+p6CknC/W5rAxp6DGsikUcan/P1zu/4xjfDsql2859U+kn/4r/PakqjHNkiWCZigYVN5btpVZK3YwO6sQWA9A37SWnNizHf07t+acQUeR3jaZeL+PuKIcWv/7/xH/w5cAaMvOBEbci7/7SEgfTjd//GGOZoxp6iwRNCMl5QGenL2GFz7fULns5C5x/OanJzDqmPahn37dsx7+1x0crkUH+OkTSP/z8Ns1fWNihiWCJk5VWbMznzcXbeGvXzpdKbRKjOOUPh24Y1x/Nny3gJN6dQi1IXzxOMx5yJkfez+cfEsEIzfGNBaWCJqwF+dt5H/nrmfnfufBrCHpqZzZP42bRvd2+3aHDaE23LIQ3rvR6XCtTXf4xRvQqX/kAjfGNCqWCJqgjTkF/OaVJWRt3w/AHeP6cd6QLnRpk3z4DfN3wfRfwg9fOf3zDL/OORNIaBGBqI0xjZUlgiYkGFSuf3kxn67aCcDoYzvy9KXHk5ocRmPuirfhvd9AeRH0PB1+/iLYrZ7GGCwRNBlbcgs59dE5ALRPSeBf155I/85H7sU+sTgHXr8cvv/QGVXqVzOh60+8DtcY04RYImjkFm/O5d2lW/nX/B8Ap4O1+XeMqWwDqFFZMXz9F0bNf8CZP3Y8nPcMtOzoccTGmKbGEkEjduv0ZbyzZCsA/Tu35rGJgxnUNRV2ZsE3z0POGmfkLQ04QysGgwemczdAsJxyfwviJjwNQy6Jcm2MMY2VJYJGasbybbyzZCsJfh+f3Hoa3dunOF/0s+6A+c85hToPgeS24Is7MAB5xSDk3UZAv3P5clsSGUNGR7cyxphGzRJBI7SvqIybX1tKUryPL//7DDq0TITVH8EHt0C+01DM1f92+vo5ku2ZnsZqjGn6LBE0MoGgMvKhzwB45KLBdGgRB/+8ADbMgeR2MP5xGHYN+GyELWNMw7BE0Ijs2FfMza8vZUB5FqM75HLBjs/h/eedlfEpcPNSSG4T3SCNMc2OJYJGYldeMWMe/pD/JP6BtMQfIR/4xl3Z6wy44h1neC5jjGlglgiirLC0nJteWULp2jmsTHL7/ek6DC58wXngK7mtJQBjjKcsEURRIKiM+/MX9Nz7Fa8mPOosPONuOO2P0Q3MGBNTLBFE0emPzaHNjyt5MdFNAr94E/qeFd2gjDExx249iYItuYX0unMm5Xuz+TDxLmfhRX+1JGCMiQpLBBG2O6+EUx+dQ4fgHuYn/dZZeMrvYfDPoxuYMSZm2aWhCNqwO58znpgLwLzkW0FxngsYcV10AzPGxDQ7I4igCX+ZhxDk445/Jk5LoV0vSwLGmKizM4IIyVy9i4KSUv6a+DTH5i1yFl797+gGZYwxWCKIiNLyIFf/YyFPxD/PGFkERx0H13wK8UnRDs0YY7y9NCQi54jIahFZJyK3h1h/tIjMEZGlIvKtiIz3Mp5oGfvUXK70z+Zn/i+dBdfPtSRgjGk0PEsEIuIHngXGAQOAy0RkQLVidwHTVfV44FLgOa/iiZZbpy9j854Cboz70Fnw+5VOV9HGGNNIeHlGMAJYp6obVLUUeB04v1oZBSrGW0wFtnkYT8S9MHc97yzZys98X9BFcmDE9ZCaHu2wjDHmIKKq3uxYZCJwjqpe685fCZyoqpOqlOkMzAbaAinAmaq6OMS+rgeuB0hLSzvh9ddfr1NM+fn5tGzZsk7b1tbOgiD//UURAKtTriMxUMDc095CfWEMNN+AIlnnxsLqHBuszrUzevToxao6LNQ6LxuLQ/WUVj3rXAa8qKpPiMgo4GURGaSqwYM2Up0GTAMYNmyYZmRk1CmgzMxM6rptbezKK+bqB50xBW5JX0NiTgEMvpTTzxjr+bGri1SdGxOrc2ywOjccLy8NZQPdqsync+iln2uA6QCq+jWQBHTwMCbPvb9sK6c8MgeAP45sye9zpjgrzpwSrZCMMeawvEwEC4E+ItJTRBJwGoNnVCvzAzAGQET64ySC3R7G5Kn1u/O55fVllAaCvH/TSUzKdnsRPeVWaN05usEZY0wNPEsEqloOTAI+Blbh3B20UkTuF5EJbrE/ANeJyHLgNeBq9arRIgJu+tcSAKZdeQJDVj4COWug1xg4894oR2aMMTXz9IEyVZ0JzKy27J4q01nAyV7GECnvLMlm9c48xg5I46x2u+DN56FFe7j4n9EOzRhjDsueLG4gT8xeA8DU0anwgtsw//MXITG27mowxjQ91ulcA9j6YxFbfyxiwpAupM28xlk48iboeVp0AzPGmDBYImgAj876HoCf9SiF7cshKRXOeTjKURljTHgsETSA95c5d8WevuJOZ8FldXvgzRhjosESQT3tLy4DYHLHebB1EQyaCN1PinJUxhgTPksE9fTqNz+QSj7X5T3rLBj/WHQDMsaYWrK7hurpkY++5+tEt4ftc5+GFu2iG5AxxtSSnRHUw/ItPzLKt5LOkgutOsOwX0U7JGOMqTVLBPVww78Wc7n/U2fGhp00xjRRlgjq6Mu1OezaV8C5/m/g2PHQvle0QzLGmDqxRFBHL3y+niv9nzgz3ZtFLxnGmBh1xEQgIskicoeIPO/O9xaRcd6H1njtKyrji7U5TIl3+xE6/vLoBmSMMfUQzhnB33EGmTnFnd8GPORZRE3AsKmfMETWOTMd+kJy2+gGZIwx9RBOIuijqg8BZQCqWkjo0cdiwqwVOygLKA+1n+UsuOSV6AZkjDH1FE4iKBWRJNxhJkWkJ1DqaVSN2F3vraAVhQzM/8rpVK5j32iHZIwx9RLOA2UPALOAdBF5CTgduNbTqBqpz1btJCe/hP+X+h2UAEOviHZIxhhTb0dMBKr6kYgsAk7CuSR0m6ru8jyyRuialxYBcFvrT50BNXuccvgNjDGmCQjnrqHZqrpbVd9X1fdUdZeIzI5EcI3JPe+vAODc/m2I270S4lMgtWuUozLGmPqr8YzAHXA+CUgTkVYcaCBuDRwdgdgajSdnr+afX28mJcHPM70Xw0bghKujHZYxxjSIw50R/AZYCfRz3yteHwPPex9a47Bhdz7P/Me5VXT+nWPwFe9zVpz+X1GMyhhjGk6NZwSq+hTwlIj8TlWfjmBMjcqjs1YD8OdLh9IqKR7WzAJ/IiS3iXJkxhjTMMJpLH5aRPoBA3AuFVUsf9XLwBqDRZtymbVyByf3bs/5Q7vCju9g5wonERhjTDNxxEQgIncBZ+FcIvoYOBv4Emj2iWDi818D8MD5g5wF797gvI+9P0oRGWNMwwvngbJLgNHAdlW9EhhCDAxo8/LXmwA4Z+BRHNOxJag6ZwMAI2+IWlzGGNPQwkkERaoaAMrdu4d2AMd4G1b0vfLNDwA8dclQZ8G6z5z3k26OUkTGGOONcH7ZLxWRNjidzy0C9gNLPI0qyorLAny/I4++aS1JTvA7C7//wHk/0c4GjDHNy2ETgYgIMEVVfwSeFZGPgdaq2qwTwZzvnQenf35CtwMLs2Y4jcStu0QpKmOM8cZhLw2pqgIfVplf19yTAMCNrzhVPLVvB2dBMAjlJdDxWJCY7XjVGNNMhdNGsEBEfuJ5JI3E0h/2Vk73O6q1M7FnLZQVwKCfRSkqY4zxTjhtBKcA14nIeqAAp6sJVdVmmRzufNe5M+jdm046sHCtOyRl+95RiMgYY7wVTiK4oK47F5FzgD8DfuCvqvpIiDIXA1NwxjtYrqq/qOvx6uvHwlJWbd/PqGPac/zR7qhjwQDMnuxMH5MRrdCMMcYz4TxZvL4uOxYRP/AsMBbIBhaKyAxVzapSpg9wB3Cyqu4VkU51OVZDeXjm9wD89owqv/y/e+vAdGLLCEdkjDHeC6eNoK5GAOtUdYOqlgKvA+dXK3Md8Kyq7gWI9jgHM7/bTqvEOEb1an9g4cbPnfffLIxOUMYY4zEvnxDuCmypMp8NnFitTF8AEZmHc/loiqrOqr4jEbkeuB4gLS2NzMzMOgWUn59f47a5xUHySsoZ1dnP3LlzAfCXF3Lqsn8RlDg+X7kN2Fan40bT4ercXFmdY4PVueGElQhEJB1nEPs5IpIIxKlqwZE2C7FMQxy/D5ABpANfiMgg97mFAxupTgOmAQwbNkwzMjLCCfsQmZmZ1LTtpFeXAEVcdcYQMo7r7Cx89RIAfKNvJ+O0uh0z2g5X5+bK6hwbrM4NJ5wRyn4NzAD+6i7qDrwfxr6zgSpPZJHOoT+ps4H3VbVMVTcCq3ESQ8St25UPwLiKJABOl9MAJ90ShYiMMSYywmkjuBkYidO1BKq6BginUXch0EdEerqjnV2Kk1Cqeg+nQztEpAPOpaIN4YXecApKyvl+Rx4/ObrKGAPBoPN+3M8hLiHSIRljTMSEkwiK3cZeoPJuoCM+Xquq5cAknK6rVwHTVXWliNwvIhPcYh8De0QkC5gD3Kaqe2pbifqa+d12AM4bUqX7iP1bnfc2MTUqpzEmBoXTRjBPRP4LSBKR0ThDWH54hG0AUNWZwMxqy+6pMq3Are4rav725UYALj+x+4GFcx5y3vucFYWIjDEmcsI5I/gvIA/4HrgF+AyY7GVQkVYeVOJ8QkKc+3GUFsDyVyH1aOhW/UYnY4xpXsI5IxiP81Tw/3odTDQs2JjLul35XDGyyiWg7cud967HWydzxphmL5wzgouBdSLyDxE5220jaDbeW+a0BVwyrEoi+PpZ5330XVGIyBhjIuuIicAdnrIv8AHwa2CDiDzvdWCR8uXaHAAGdml9YOHmeZDQCjr2jVJUxhgTOWE9UKaqJSLyPlCE8wTwxUCzGKqruCxAp1aJ+HxVLgHFp0B7G4DGGBMbwnmg7EwR+SuwHrgC+CdwlNeBRcK+ojJ25ZUwblCV6ix/HfZnQ9rA6AVmjDERFM4ZwQ04Hcb9VlWLPI4nopZsdgah6ZPW6sDC92503odeHoWIjDEm8sLphnpiJAKJhs/X7gZgUNdUZ0FpIWgQepwK3YZHMTJjjImcGhOBiMxV1dNFZC8HdxZXMUJZO8+j89g/5m0iMc7HkHQ3EVTcNtr7zOgFZYwxEXa4M4LR7nuHSAQSaXsLnF4z+ndujVQ8K5C9wHnvPDhKURljTOTV2Fisqm6va/xNVQNVX8DfIhOed1bt2A9U619o1QfOe/qIKERkjDHREc4DZQf9PHYfKGvyF9CztjmJYGi3Kj2OZi+ENt1tSEpjTEypMRGIyH+77QODRSTXfe0FdlOtI7mmaP1uZ1ydHu1bOAsKc533/udFKSJjjImOw50RPAp0BJ5y3zsCHVS1nareFongvDR3tTM8cvuWic6CPKcrajoPjVJExhgTHYdrLO6tqmtF5GWg8umqioZVVf3W49g8EwgqewvLDu5WYscK590GoTHGxJjDJYLbgWuAZ0OsU+A0TyKKgEWbcikqC3DJ8CojaW5b6rx3sieKjTGxpcZEoKrXuO+nRi6cyPh+Rx4AvTtVaRTeMMd5b98rChEZY0z0hNPX0EUi0sqdvl1EpovIEO9D847f7WCud8cqiWD395B2nI0/YIyJOeHcPjpFVfNE5CTgPOAN4AVvw/LWym37AA6MSJbvdDVB2+41bGGMMc1XOIkg4L6fCzynqm8Did6F5L38EqdKLRPdK2NrZzvvfc+JUkTGGBM94fQ+ul1EngXGASeISALhJZBGa+XWfXRrl0yc363G5q+c92MyohWSMcZETbhDVc4FxqvqXpy+h273NCoPBYPKhpwCurVtcWBhRbtAm26hNzLGmGYsnKEq84EsIENEbgDaqupHnkfmkf3FZYDT2VylDXOhY78oRWSMMdEVzl1Dk4DpwNHua7qI3OR1YF7ZlVcCQN80946h8lLY9wOkdIxiVMYYEz3htBFcD4xwzwwQkYeAr4DnvAzMK2t2Os8QpFdcGipyRimjxylRisgYY6IrnDYCAcqqzJe5y5qksoDTu3brpHhnQbFzKymtbbB6Y0xsCueM4GVgvoi8jZMALgBe8jQqD7l5gDYt3ESw8zvnPS45OgEZY0yUhTNm8aMiMgeo6GriBlVd6G1Y3il3M0HF08VsyHTe+9jwlMaY2BTOGQFAifsKuu9NVtZ2Z0CayqeKl/wTWrSH5LZRjMoYY6InnLuGJgOvAZ2BdOBVEbnD68C8sjHHGZCmfUoCfPeWszDNehw1xsSucM4IrgBOUNVCABF5EFgMPOxlYF5ZtX0/8X5Bti6Bt69xFmbcGd2gjDEmisK5a2gzByeMOGBDODsXkXNEZLWIrBORGp9GFpGJIqIiMiyc/daHT4Qh6W1gy3xnwc9fgu6jvD6sMcY0WuEkgkJgpYj8VUT+D/gO+FFEnhSRJ2vayB3kvqKPogHAZSIyIES5VsDNwDd1qUBtBFXZlVdClzbJsGmes/DYcV4f1hhjGrVwLg39231VmB/mvkcA61R1A4CIvA6cj9NdRVUP4IyP/Mcw91tnbqej9OiQAisrhqZs0h2pGmNMvYVz++jf6rjvrsCWKvPZwIlVC4jI8UA3Vf1QRGpMBCJyPc4TzqSlpZGZmVmngHbsLQCELZs3Edi/nZLkziyo476aivz8/Dp/Xk2V1Tk2WJ0bTri3j9ZFqKePtXKliA94Crj6SDtS1WnANIBhw4ZpRkZGnQLa9uF/gCLGDeqIP7uUFn1Oo677aioyMzObfR2rszrHBqtzw/FyXIFsoGq/zunAtirzrYBBQKaIbAJGAjO8bDAuDTh5qOPeZc6CPmd5dShjjGkywk4EIlLbi+kLgT4i0tMdzOZSYEbFSlXdp6odVLWHqvbAaXuYoKqLanmcsOWVOomgy6Z3nAVdT/DqUMYY02SE80DZCBH5Dljrzg8Rkf850naqWg5MAj4GVgHTVXWliNwvIhPqGXediDsATdrOz50F7XpGIwxjjGlUwmkjeAZnvOL3AFR1uYiMDmfnqjoTmFlt2T01lM0IZ5/1EVTFXzEEc1tLAsYYA+FdGvKp6uZqywIhSzZyQYVWFDoz9vyAMcYA4Z0RbBGREYC6D4n9FljjbVjeyC1WjhJ3IJo2R0c3GN5/bXQAABuHSURBVGOMaSTCOSO4EbgVZ5jKnTh399zoZVBe8Qt0l53OTKvO0Q3GGGMaiXAeKNuFc8dPk5dbrAdm2veKXiDGGNOIHDERuP0LafXlqnq9JxF5KAiky25nxhcf1ViMMaaxCKeN4NMq00nAhRzcdUSTIUBpRZUTW0Y1FmOMaSzCuTT0RtV5EXkZ+MSziDykCvHintzEt4huMMYY00jUpYuJnkD3hg4kEhSIE/fOV58/qrEYY0xjEU4bwV4OtBH4gFygxkFmGrOgQkrFkMs+L/vbM8aYpuOw34bi9MkwBNjqLgqq6iENx01FUKGz7HFm/AnRDcYYYxqJw14acr/031XVgPtqskkAYHdRkAKSnBm/3TVkjDEQXhvBAhH5ieeRRECSX+iu2yAxNdqhGGNMo1HjpSERiXN7ED0FuE5E1gPOEF/OyUKTTA4anwIl+6IdhjHGNBqHayNYAPwEuCBCsXhOAR9B6HBstEMxxphG43CJQABUdX2EYvGcovhE7dZRY4yp4nCJoKOI3FrTSlV90oN4PKUKfoIgXo7QaYwxTcvhEoEfaEnoQeibJFXwoZYIjDGmisMlgu2qen/EIokABXoEfwCxLqiNMabC4X4aN5szgQoK7POlQkFOtEMxxphG43CJYEzEooiQrflBRBU69Yt2KMYY02jUmAhUNTeSgURCq3hBgwEQu2vIGGMqxFSr6bb8IC3isdtHjTGmiphKBIlxQpvyHLtryBhjqoipvphFlVT2Q2lBtEMxxphGI6Z+GsdR5ky0sttHjTGmQkwlgnjKnYm0AdENxBhjGpGYSgRt1e11tKw4uoEYY0wjElOJQAg6E+16RjcQY4xpRGIqEfgrEoHdPmqMMZViKxGU5jkTNnC9McZU8jQRiMg5IrJaRNaJyO0h1t8qIlki8q2IfCYi3b2Mp63kOxOWCIwxppJniUBE/MCzwDhgAHCZiFS/XWcpMExVBwNvAY96FQ9AB3Ebi1PTvTyMMcY0KV6eEYwA1qnqBlUtBV4Hzq9aQFXnqGqhOzsf8PQbWis6VE3p6OVhjDGmSfHyGklXYEuV+WzgxMOUvwb4KNQKEbkeuB4gLS2NzMzMWgejqvSRbAC+nL+Q8vhWtd5HU5Sfn1+nz6spszrHBqtzw/EyEYQaz0BDFhS5AhgGnB5qvapOA6YBDBs2TDMyMmodTCColP3nAQBOGXterbdvqjIzM6nL59WUWZ1jg9W54XiZCLKBblXm04Ft1QuJyJnAZOB0VS3xKpigKnkkE5B47OZRY4w5wMs2goVAHxHpKSIJwKXAjKoFROR44AVggqru8jAWgqq0pJi8pKO8PIwxxjQ5niUCVS0HJgEfA6uA6aq6UkTuF5EJbrHHgJbAmyKyTERm1LC7eisPKKf4vsMXLPfqEMYY0yR5ekO9qs4EZlZbdk+V6TO9PH5V+4vLKNVUEuLb0TpSBzXGmCYgZp4sLitXgghlqT2iHYoxxjQqMZMINufaYDTGGBNKzCSCwtIAPX07SUm07iWMMaaqmEkEm/cUUKiJJJbnRTsUY4xpVGImEbTxFdFCSpAux0c7FGOMaVRiJhHElTk9jwZadYlyJMYY07jEzAVzn7rPD/gTohuIMR4rKysjOzub4uLmPSRramoqq1atinYYERVOnZOSkkhPTyc+Pj7s/cZMIkgszXWnQnZ3ZEyzkZ2dTatWrejRowciobr8ah7y8vJo1So2Oo+scKQ6qyp79uwhOzubnj3DH5I3Zi4Ntc7fBIAmd4huIMZ4rLi4mPbt2zfrJGBCExHat29f67PBmEkE/qDTn12gTY/oBmJMBFgSiF11+bePmURQcUlIE2PrVNIYY44khhKBy34oGeO5HTt2cOmll9KrVy8GDBjA+PHjWbNmDZs2bUJE+J//+Z/KspMmTeLFF18E4Oqrr6Zr166UlDhn8Dk5OfTo0SPkMYqKijj99NMJBAJeV6fOZs2axbHHHkvv3r155JFHQpbZvHkzY8aMYfDgwWRkZJCdnX3Q+v3799O1a1cmTZpUuezMM89k7969DRZn7CQCt43Y8oAx3lJVLrzwQjIyMli/fj1ZWVk89NBD7Ny5E4BOnTrx5z//mdLS0pDb+/1+/v73vx/xOC+//DIXXXQRfn94I4yoKsFgMPyK1FMgEOA3v/kNH330EVlZWbz22mtkZWUdUu6Pf/wjV111Fd9++y333HMPd9xxx0Hr7777bk4//eAxu6688kqee+65Bos1Zu4aOsBSgYkd932wkqxt+xt0nwO6tObe8wbWuH7OnDnEx8dzww03VC4bOnQoAJs2baJjx46cfPLJvPTSS1x33XWHbP+73/2Op556KuS6qqZPn84bb7wBOEM4nn/++ezdu5eysjKmTp3K+eefz6ZNmxg3bhyjR4/m66+/5r333mP16tXce++9lJSU0KtXL/7xj3/QsmVL7r//fj744AOKioo46aSTeOGFF+rV1rJgwQJ69+7NMcccA8Cll17K+++/z4ABAw4ql5WVxVNPPQXA6NGjueCCCyrXLV68mJ07d3LOOeewaNGiyuUTJkzg1FNPZfLkyXWOr6oYOiNwTgmsEc0Yb61YsYITTjjhsGVuv/12nnjiiZCXdY4++mhOOeUUXn755Rq3Ly0tZdOmTZWXjZKSknj33XdZsmQJc+bM4Q9/+APq/p9fvXo1V111FUuXLiUlJYWpU6fy6aefsmTJEoYNG8aTTz4JOJeoFi5cyIoVKygqKuLDDz885LivvPIKQ4cOPeQ1ceLEQ8pu3bqVbt0ODNKYnp7O1q1bDyk3ZMgQ3n77bQDeffdd8vLy2LNnD8FgkD/84Q889thjh2zTtm1bSkpK2LNnT42fUW3E0BlBxSmhJQITOw73yz2aevbsyYgRI3j11VdDrr/zzjuZMGECP/3pT0Ouz8nJITU1tXJeVbnzzjv5/PPP8fl8bN26tfJSVPfu3Rk5ciQA8+fPJysri5NPPhlwEsqoUaMA50zm0UcfpbCwkNzcXAYOHMh55x08vvnll1/O5ZdfHlYdKxJRVaF+iD7++OOV7SSnnXYaXbt2JS4ujueee47x48cflEyq6tSpE9u2baN9+/ZhxXM4MZQIHHZGYIy3Bg4cyFtvvXXEcnfeeScTJ07ktNNOO2Rd7969GTp0KNOnTw+5bXJycmWDMji/1Hfv3s3ixYuJj4+nR48elffSp6SkVJZTVcaOHctrr7120P6Ki4u56aabWLRoEd26dWPKlCkh78V/5ZVXQv5C79279yF1Tk9PZ8uWLZXz2dnZdOlyaBc3Xbp04Z133gGcS1xvv/02qampfP3113zxxRc899xz5OfnU1paSkJCQuUZTHFxMcnJySE/n9qKoUtD0Q7AmNhwxhlnUFJSwv/93/9VLlu4cCFz5849qFy/fv0YMGBAyEswAJMnT+bxxx8Pua5t27YEAoHKL+t9+/bRqVMn4uPjmTNnDps3bw653ciRI5k3bx7r1q0DoLCwkDVr1lTup0OHDuTn59eYyC6//HKWLVt2yCtU+eHDh7N27Vo2btxIaWkpr7/+OhMmTDikXE5OTmUj9sMPP8yvf/1rwEk6P/zwA5s2beLxxx/nqquu4r777gOchLZjx44a76iqrdhJBJXsjMAYL4kI7777Lp988gm9evVi4MCBTJkyJeSv4cmTJx9yu2SFgQMH8pOf/KTG45xxxhl8+eWXgPMFvWjRIoYNG8Yrr7xCv379Qm7TsWNHXnzxRS677DIGDx7MyJEj+f7772nTpg3XXXcdxx13HBdccAHDhw+vQ80PFhcXx1/+8hfOPvts+vfvz8UXX8zAgc6lunvuuYcZM5wh2jMzMzn22GPp27cvO3fuDKsBePHixYwcOZK4uAa6qKOqTep1wgknaF18/eqDqve21h93b6vT9k3VnDlzoh1CxMV6nbOysqIXSAR98cUXesUVV0Q7jIjav3+/qqrefPPN+umnn9ZYLtTfALBIa/hejZ0zgoqGG2sjMKZZGDJkCKNHj27UD5R5ZdCgQYwZM6bB9hd7jcUxlPuMae4qrqfHmiM9Y1FbMfStaK3FxhgTSgwlAof67NKQMcZUFTOJQEM83GGMMSaGEkEFe6DMGGMOFjOJQKyNwJiI8fv9DB06lIEDBzJkyBCefPLJyoemMjMzERE++OCDyvLnnnsumZmZAGRkZDBs2LDKdYsWLSIjIyPkcbZv3865557rWT0awksvvUSfPn3o06cPL730Usgyy5cvZ9SoURx33HGcd9557N/vdBS4YMGCyv6MhgwZUvmZlZaWctppp1FeXt4gMcZMIqhoLBZ7oMwYzyUnJ7Ns2TJWrlzJJ598wsyZMyufigWn+4UHH3ywxu137drFRx99dMTjPPnkk7W6gybSt5rm5uZy33338c0337BgwQLuu+++kOMIXHvttTzyyCN89913XHjhhZXdWAwaNIhFixaxbNkyZs2axS233EJ5eTkJCQmMGTOmsvfV+oqd20crTgjs0pCJJR/dDju+a9h9HnUcjAs9yEoonTp1Ytq0aQwfPpwpU6YAzjMAZWVlfPLJJ4wdO/aQbW677TamTp3KuHHjDrvvt99+m6lTpwJOF9dXXnklBQUFAPzlL3/hpJNOIjMzk/vuu4/OnTuzbNkysrKy+Ne//sUzzzxDaWkpJ554Is899xx+v58bb7yRhQsXUlRUxMSJEw9KXnXx8ccfM3bsWNq1awfA2LFjmTVrFpdddtlB5VavXl3Z59LYsWM5++yzeeCBB2jRokVlmeLi4oMubV9wwQXccccdYXeCdzixd0ZgecCYiDvmmGMIBoPs2rWrctldd91V+SVe3ahRo0hMTGTOnDk17nPTpk20bduWxMREwEk4n3zyCUuWLOGNN97g5ptvriy7YMECHnzwQbKysli1ahVvvPEG8+bNY9myZfj9fl555RUAHnzwQRYtWsS3337L3Llz+fbbbw857mOPPRayK+qqx6sQblfUgwYNquxy4s033zyos7pvvvmGgQMHctxxx/H0009XdisxaNAgFi5cWOPnUxsxc0ZwoIXAMoGJIbX45e616nfunXrqqQB88cUXIctXJIo//elPIdfv3LmTjh07Vs6XlZUxadKkyi/3NWvWVK4bMWIEPXv2BOCzzz5j8eLFlf0JFRUV0alTJ8AZ7GbatGmUl5ezfft2srKyGDx48EHHve2227jtttvqVGcIfcPK3//+d26++Wbuv/9+JkyYQEJCQuW6E088kZUrV7Jq1SquuOIKLrroIpKSkvD7/SQkJJCXl0erVvUbi93TRCAi5wB/BvzAX1X1kWrrE4F/AicAe4BLVHWTJ7FUthEYYyJtw4YN+P1+OnXqxKpVqyqXT548mQcffDBk52lnnHEGd999N/Pnzw+5z6SkpIO6in7qqadIS0tj+fLlBINBkpKSKtdV74r6l7/8JQ8//PBB+9u4cSOPP/44CxcupG3btlx99dUhu6J+7LHHKs8gqjrttNN45plnDlqWnp5e2QgOTlfUoRq++/Xrx+zZswFYs2YN//73vw8p079/f1JSUlixYkVlY3pJSclB9awrzy4NiYgfeBYYBwwALhORAdWKXQPsVdXewFNA6NTfEKyvIWOiYvfu3dxwww1MmjTpkF/DZ511Fnv37mX58uUht508eTKPPvpoyHW9e/dm06ZNlfP79u2jc+fO+Hw+Xn755RobhseMGcNbb71VeZkqNzeXzZs3s3//flJSUkhNTWXnzp01NlbfdtttIbuirp4EAM4++2xmz57N3r172bt3L7Nnz+bss88+pFxFLMFgkKlTp1YO87lx48bKO4M2b97M2rVrK7ue3rNnDx07diQ+Pj5knLXhZRvBCGCdqm5Q1VLgdeD8amXOByrup3oLGCMe3+hvzxEY472ioqLK20fPPPNMzjrrLO69996QZQ/XFfX48eMPuvxTVUpKCr169aocW+Cmm27ipZdeYuTIkaxZs+ags4CqBgwYwNSpUznrrLMYPHgwY8eOZfv27QwZMoTjjz+egQMH8utf/7pyFLP6aNeuHXfffTfDhw9n+PDh3HPPPZUNx9dee23lOMSvvfYaffv2pV+/fnTp0oVf/epXAHz55ZcMGTKEoUOHcuGFF/Lkk0/SoUMHwBlRbfz48fWOEUC8euJWRCYC56jqte78lcCJqjqpSpkVbplsd369Wyan2r6uB64HSEtLO+H111+vdTz71n9Nh+1zKDrxD8TFJ9a1Wk1Ofn4+LVu2jHYYERXrdU5NTaV3795Rjsh7gUCAmTNnsnTpUu65555ohxMRgUAAv98POGMwTJkyhT59+hxSbt26dezbt++gZaNHj16sqsMOKYy3bQShfnpXzzrhlEFVpwHTAIYNG6Y1PVxyWBkZZGaO4sy6bNuEZWZm1vgwTnMV63VetWpVvRsPm4K8vDx+8YtfUFhYGBP1BSobhktLS5k4cWKNA/ckJSVx/PHHh71fLy8NZQNVR11OB7bVVEZE4oBUINfDmIwxzcy1114b7RAiLiEhgauuuqrB9udlIlgI9BGRniKSAFwKzKhWZgbwS3d6IvAftd7hjKk3+28Uu+ryb+9ZIlDVcmAS8DGwCpiuqitF5H4RqRjB+W9AexFZB9wK3O5VPMbEiqSkJPbs2WPJIAapKnv27Kn1LaWePkegqjOBmdWW3VNluhj4uZcxGBNr0tPTyc7OZvfu3dEOxVPFxcUNcg99UxJOnZOSkkhPT6/VfmPmyWJjYkV8fHzlU7TNWWZmZq0aRJsDr+ocQ30NGWOMCcUSgTHGxDhLBMYYE+M8e7LYKyKyG9hcx807ADlHLNW8WJ1jg9U5NtSnzt1VNWR/HU0uEdSHiCyq6RHr5srqHBuszrHBqzrbpSFjjIlxlgiMMSbGxVoimBbtAKLA6hwbrM6xwZM6x1QbgTHGmEPF2hmBMcaYaiwRGGNMjGuWiUBEzhGR1SKyTkQO6dFURBJF5A13/Tci0iPyUTasMOp8q4hkici3IvKZiHSPRpwN6Uh1rlJuooioiDT5Ww3DqbOIXOz+W68UkVcjHWNDC+Nv+2gRmSMiS92/74YZvzFKROTvIrLLHcEx1HoRkWfcz+NbEQk9Ok1tqGqzegF+YD1wDJAALAcGVCtzE/C8O30p8Ea0445AnUcDLdzpG2Ohzm65VsDnwHxgWLTjjsC/cx9gKdDWne8U7bgjUOdpwI3u9ABgU7TjrmedTwN+AqyoYf144COcER5HAt/U95jN8YxgBLBOVTeoainwOnB+tTLnAy+5028BY6Rpj2p/xDqr6hxVLXRn5+OMGNeUhfPvDPAA8ChQHMngPBJOna8DnlXVvQCquivCMTa0cOqsQGt3OpVDR0JsUlT1cw4/UuP5wD/VMR9oIyKd63PM5pgIugJbqsxnu8tCllFnAJ19QPuIROeNcOpc1TU4vyiasiPWWUSOB7qp6oeRDMxD4fw79wX6isg8EZkvIudELDpvhFPnKcAVIpKNM/7JbyMTWtTU9v/7ETXH8QhC/bKvfo9sOGWakrDrIyJXAMOA0z2NyHuHrbOI+ICngKsjFVAEhPPvHIdzeSgD56zvCxEZpKo/ehybV8Kp82XAi6r6hIiMAl526xz0PryoaPDvr+Z4RpANdKsyn86hp4qVZUQkDud08nCnYo1dOHVGRM4EJgMTVLUkQrF55Uh1bgUMAjJFZBPOtdQZTbzBONy/7fdVtUxVNwKrcRJDUxVOna8BpgOo6tdAEk7nbM1VWP/fa6M5JoKFQB8R6SkiCTiNwTOqlZkB/NKdngj8R91WmCbqiHV2L5O8gJMEmvp1YzhCnVV1n6p2UNUeqtoDp11kgqouik64DSKcv+33cG4MQEQ64Fwq2hDRKBtWOHX+ARgDICL9cRJBcx6ncwZwlXv30Ehgn6pur88Om92lIVUtF5FJwMc4dxz8XVVXisj9wCJVnQH8Def0cR3OmcCl0Yu4/sKs82NAS+BNt138B1WdELWg6ynMOjcrYdb5Y+AsEckCAsBtqronelHXT5h1/gPwfyLye5xLJFc35R92IvIazqW9Dm67x71APICqPo/TDjIeWAcUAr+q9zGb8OdljDGmATTHS0PGGGNqwRKBMcbEOEsExhgT4ywRGGNMjLNEYIwxMc4SgWm0RCQgIsuqvHocpmyPmnprjDQRGSYiz7jTGSJyUpV1N4jIVRGMZWhT743TeK/ZPUdgmpUiVR0a7SBqy31oreLBtQwgH/jKXfd8Qx9PROLcPrNCGYrTpcjMhj6uaT7sjMA0Ke4v/y9EZIn7OilEmYEissA9i/hWRPq4y6+osvwFEfGH2HaTiPzJLbdARHq7y7uLM45DxXgOR7vLfy4iK0RkuYh87i7LEJEP3TOYG4Dfu8c8VUSmiMgfRaS/iCyoVq9v3ekTRGSuiCwWkY9D9SwpIi+KyJMiMgf4k4iMEJGvxOmT/ysROdZ9Evd+4BL3+JeISIo4/d0vdMuG6rHVxJpo971tL3vV9MJ5MnaZ+3rXXdYCSHKn++A8XQrQA7f/duB/gMvd6QQgGegPfADEu8ufA64KccxNwGR3+irgQ3f6A+CX7vSvgffc6e+Aru50G/c9o8p2U4A/Vtl/5bxbr2Pc6f8G7sJ5gvQroKO7/BKcp2mrx/ki8CHgd+dbA3Hu9JnA2+701cBfqmz3EHBFRbzAGiAl2v/W9oruyy4NmcYs1KWheOAvIjIUJ1H0DbHd18BkEUkH3lHVtSIyBjgBWOh2sZEM1NTn0mtV3p9yp0cBF7nTL+OMcQAwD3hRRKYD79SmcjgdpV0MPILzhX8JcCxOZ3mfuHH6gZr6kXlTVQPudCrwknv2o7hdEoRwFjBBRP7ozicBRwOrahm7aUYsEZim5vfATmAIzqXNQwacUdVXReQb4KfAxyJyLU7XvS+p6h1hHENrmD6kjKreICInusda5iaocL2B0/fTO86udK2IHAesVNVRYWxfUGX6AWCOql7oXpLKrGEbAX6mqqtrEadp5qyNwDQ1qcB2dfqavxLnF/NBROQYYIOqPoPTU+Ng4DNgooh0csu0k5rHbb6kyvvX7vRXHOic8HLgS3c/vVT1G1W9B8jh4O6BAfJwusQ+hKquxzmruRsnKYDTbXRHcfrVR0TiRWRgDXFWlQpsdaevPszxPwZ+K+7phji90poYZ4nANDXPAb8Ukfk4l4UKQpS5BFghIsuAfjjD+mXhXIOf7TbKfgLUNLxfontGcQvOGQjAzcCv3G2vdNcBPCYi37m3rn6OM6ZuVR8AF1Y0Foc41hvAFRzoT78Up2v0P4nIcpx2hEMaxEN4FHhYROZxcHKcAwyoaCzGOXOIB751Y34gjH2bZs56HzWmCnEGsRmmqjnRjsWYSLEzAmOMiXF2RmCMMTHOzgiMMSbGWSIwxpgYZ4nAGGNinCUCY4yJcZYIjDEmxv1/BXPHo8IEPT0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#import glob\n",
    "import matplotlib.pyplot as plt\n",
    "#import cv2\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "#import models\n",
    "from keras.callbacks import ReduceLROnPlateau, ModelCheckpoint\n",
    "\n",
    "\n",
    "y_pred_test = dnn_model.predict(test_set)\n",
    "y_pred_test = y_pred_test.ravel()\n",
    "y_true_test = testy.values.ravel()\n",
    "fpr_test, tpr_test, thresholds_test = roc_curve(y_true_test, y_pred_test)\n",
    "auc_test = auc(fpr_test, tpr_test)\n",
    "\n",
    "\n",
    "y_pred_val = keras_model.predict(test_set_cnn)\n",
    "y_pred_val = y_pred_val.ravel()\n",
    "y_true_val = testy.values.ravel()\n",
    "fpr_val, tpr_val, thresholds_val = roc_curve(y_true_val, y_pred_val)\n",
    "auc_val = auc(fpr_val, tpr_val)\n",
    "\n",
    "\n",
    "\n",
    "plt.plot(fpr_val, tpr_val, label= 'CNN (area = {:.2f})'.format(auc_val))\n",
    "plt.plot(fpr_test, tpr_test, label= 'DNN (area = {:.2f})'.format(auc_test))\n",
    "plt.xlabel('False positive rate')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.title('ROC curve')\n",
    "plt.grid()\n",
    "plt.legend(loc='best')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19584/19584 [==============================] - 1s 42us/sample - loss: 0.3456 - acc: 0.8648\n"
     ]
    }
   ],
   "source": [
    "_, accuracy_test = keras_model.evaluate(test_set_cnn, testy, batch_size=128, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19584/19584 [==============================] - 0s 8us/sample - loss: 0.3616 - acc: 0.8707\n"
     ]
    }
   ],
   "source": [
    "_, accuracy_dnn = dnn_model.evaluate(test_set, testy, batch_size=128, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_34 (Dense)             (None, 64)                1664      \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dense_36 (Dense)             (None, 256)               33024     \n",
      "_________________________________________________________________\n",
      "dense_37 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 141,825\n",
      "Trainable params: 141,825\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "dnn_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.61      0.84      0.71      5051\n",
      "         1.0       0.94      0.81      0.87     14533\n",
      "\n",
      "    accuracy                           0.82     19584\n",
      "   macro avg       0.77      0.83      0.79     19584\n",
      "weighted avg       0.85      0.82      0.83     19584\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(testy, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.58      0.92      0.71      6001\n",
      "         1.0       0.97      0.78      0.86     18530\n",
      "\n",
      "    accuracy                           0.81     24531\n",
      "   macro avg       0.77      0.85      0.79     24531\n",
      "weighted avg       0.87      0.81      0.83     24531\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(valy, y_pred_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dnn_model.save(\"final_model/keras2_dnn.h5\")\n",
    "model_json = dnn_model.to_json()\n",
    "with open(\"final_model/keras2_dnn.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}