TF Sequential with FFT input.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import cupy as cp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import soundfile as sf\n",
    "import gc\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "from tensorflow.keras import layers,regularizers,Sequential,backend,callbacks,optimizers,metrics,losses\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from skmultilearn.model_selection import IterativeStratification\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "testfiles = glob.glob( 'data/test/*.flac' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8997, 1000), (1992, 1000), (8997, 26))"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train = np.load(\"savedata/TF1/TRAIN.npz\")\n",
    "TRAIN = data_train['arr_0']\n",
    "\n",
    "data_test = np.load(\"savedata/TF1/TEST.npz\")\n",
    "TEST = data_test['arr_0']\n",
    "\n",
    "y_train_all_classes = pd.read_hdf(\"savedata/TF1/y_train.hd5\")\n",
    "\n",
    "TRAIN.shape, TEST.shape, y_train_all_classes.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recording_id</th>\n",
       "      <th>species_id</th>\n",
       "      <th>s0</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>s6</th>\n",
       "      <th>s7</th>\n",
       "      <th>...</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "      <th>s22</th>\n",
       "      <th>s23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>003bec244</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>006ab765f</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>007f87ba2</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0099c367b</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>009b760e6</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  recording_id  species_id  s0  s1  s2  s3  s4  s5  s6  s7  ...  s14  s15  \\\n",
       "0    003bec244          14   0   0   0   0   0   0   0   0  ...    1    0   \n",
       "1    006ab765f          23   0   0   0   0   0   0   0   0  ...    0    0   \n",
       "2    007f87ba2          12   0   0   0   0   0   0   0   0  ...    0    0   \n",
       "3    0099c367b          17   0   0   0   0   0   0   0   0  ...    0    0   \n",
       "4    009b760e6          10   0   0   0   0   0   0   0   0  ...    0    0   \n",
       "\n",
       "   s16  s17  s18  s19  s20  s21  s22  s23  \n",
       "0    0    0    0    0    0    0    0    0  \n",
       "1    0    0    0    0    0    0    0    1  \n",
       "2    0    0    0    0    0    0    0    0  \n",
       "3    0    1    0    0    0    0    0    0  \n",
       "4    0    0    0    0    0    0    0    0  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_all_classes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define Model\n",
    "input_shape = 1000\n",
    "output_shape = 24\n",
    "def build_model():\n",
    "    model = tf.keras.Sequential([\n",
    "        tf.keras.layers.Input(input_shape),\n",
    "        tf.keras.layers.BatchNormalization(),\n",
    "        tf.keras.layers.Dropout(0.2),\n",
    "        tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation=\"relu\")),\n",
    "        tf.keras.layers.BatchNormalization(),\n",
    "        tf.keras.layers.Dropout(0.2),\n",
    "        tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation=\"relu\")),\n",
    "        tf.keras.layers.BatchNormalization(),\n",
    "        tf.keras.layers.Dropout(0.5), \n",
    "        tfa.layers.WeightNormalization(tf.keras.layers.Dense(500, activation=\"relu\")),\n",
    "        tf.keras.layers.BatchNormalization(),\n",
    "        tf.keras.layers.Dropout(0.2),        \n",
    "        tfa.layers.WeightNormalization(tf.keras.layers.Dense(output_shape, activation=\"sigmoid\"))\n",
    "        ])\n",
    "    model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam(), sync_period=10),\n",
    "                  loss=losses.BinaryCrossentropy(label_smoothing=0.000000001),metrics=['AUC']\n",
    "                  )\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_cols = []\n",
    "for tgt in range(24):\n",
    "    target_cols.append('s'+str(tgt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_TARGETS = y_train_all_classes[tar_col].values\n",
    "TRAIN_TARGETS = np.array(TRAIN_TARGETS,dtype='float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.DataFrame({\"recording_id\" : [path[10:19] for path in testfiles]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold ==> 0\n",
      "WARNING:tensorflow:`epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "Epoch 1/40\n",
      "57/57 [==============================] - 11s 141ms/step - loss: 0.7198 - auc: 0.5146 - val_loss: 0.6154 - val_auc: 0.5683\n",
      "Epoch 2/40\n",
      "57/57 [==============================] - 9s 151ms/step - loss: 0.6080 - auc: 0.5610 - val_loss: 0.4652 - val_auc: 0.5366\n",
      "Epoch 3/40\n",
      "57/57 [==============================] - 9s 159ms/step - loss: 0.4183 - auc: 0.5519 - val_loss: 0.2796 - val_auc: 0.5511\n",
      "Epoch 4/40\n",
      "57/57 [==============================] - 10s 173ms/step - loss: 0.2354 - auc: 0.5402 - val_loss: 0.1332 - val_auc: 0.5407\n",
      "Epoch 5/40\n",
      "57/57 [==============================] - 9s 167ms/step - loss: 0.1369 - auc: 0.5760 - val_loss: 0.0934 - val_auc: 0.5505\n",
      "Epoch 6/40\n",
      "57/57 [==============================] - 10s 172ms/step - loss: 0.0921 - auc: 0.6004 - val_loss: 0.0714 - val_auc: 0.5423\n",
      "Epoch 7/40\n",
      "57/57 [==============================] - 10s 175ms/step - loss: 0.0701 - auc: 0.6049 - val_loss: 0.0570 - val_auc: 0.5641\n",
      "Epoch 8/40\n",
      "57/57 [==============================] - 14s 239ms/step - loss: 0.0567 - auc: 0.6285 - val_loss: 0.0468 - val_auc: 0.5686\n",
      "Epoch 9/40\n",
      "57/57 [==============================] - 10s 170ms/step - loss: 0.0500 - auc: 0.6435 - val_loss: 0.0464 - val_auc: 0.5868\n",
      "Epoch 10/40\n",
      "57/57 [==============================] - 8s 149ms/step - loss: 0.0440 - auc: 0.6515 - val_loss: 0.0415 - val_auc: 0.5807\n",
      "Epoch 11/40\n",
      "57/57 [==============================] - 10s 177ms/step - loss: 0.0411 - auc: 0.6936 - val_loss: 0.0389 - val_auc: 0.6040\n",
      "Epoch 12/40\n",
      "57/57 [==============================] - 10s 169ms/step - loss: 0.0397 - auc: 0.6966 - val_loss: 0.0378 - val_auc: 0.5941\n",
      "Epoch 13/40\n",
      "57/57 [==============================] - 11s 193ms/step - loss: 0.0367 - auc: 0.7147 - val_loss: 0.0363 - val_auc: 0.6165\n",
      "Epoch 14/40\n",
      "57/57 [==============================] - 13s 222ms/step - loss: 0.0368 - auc: 0.7353 - val_loss: 0.0370 - val_auc: 0.6200\n",
      "Epoch 15/40\n",
      "57/57 [==============================] - 12s 209ms/step - loss: 0.0357 - auc: 0.7399 - val_loss: 0.0346 - val_auc: 0.6311\n",
      "Epoch 16/40\n",
      "57/57 [==============================] - 11s 194ms/step - loss: 0.0344 - auc: 0.7695 - val_loss: 0.0352 - val_auc: 0.6211\n",
      "Epoch 17/40\n",
      "57/57 [==============================] - 9s 160ms/step - loss: 0.0326 - auc: 0.7729 - val_loss: 0.0348 - val_auc: 0.6457\n",
      "Epoch 18/40\n",
      "57/57 [==============================] - 9s 159ms/step - loss: 0.0329 - auc: 0.7795 - val_loss: 0.0350 - val_auc: 0.6472\n",
      "Epoch 19/40\n",
      "57/57 [==============================] - 12s 216ms/step - loss: 0.0312 - auc: 0.7883 - val_loss: 0.0342 - val_auc: 0.6589\n",
      "Epoch 20/40\n",
      "57/57 [==============================] - 11s 189ms/step - loss: 0.0309 - auc: 0.8023 - val_loss: 0.0339 - val_auc: 0.6696 ETA: 1s - loss: 0.0308 - auc:\n",
      "Epoch 21/40\n",
      "57/57 [==============================] - 11s 199ms/step - loss: 0.0311 - auc: 0.8086 - val_loss: 0.0339 - val_auc: 0.6680\n",
      "Epoch 22/40\n",
      "57/57 [==============================] - 11s 195ms/step - loss: 0.0309 - auc: 0.8144 - val_loss: 0.0340 - val_auc: 0.6652\n",
      "Epoch 23/40\n",
      "57/57 [==============================] - 11s 189ms/step - loss: 0.0317 - auc: 0.8054 - val_loss: 0.0338 - val_auc: 0.6721\n",
      "Epoch 24/40\n",
      "57/57 [==============================] - 9s 161ms/step - loss: 0.0325 - auc: 0.8155 - val_loss: 0.0338 - val_auc: 0.6746\n",
      "Epoch 25/40\n",
      "57/57 [==============================] - 10s 172ms/step - loss: 0.0311 - auc: 0.8092 - val_loss: 0.0337 - val_auc: 0.6733\n",
      "Epoch 26/40\n",
      "57/57 [==============================] - 9s 153ms/step - loss: 0.0316 - auc: 0.8116 - val_loss: 0.0337 - val_auc: 0.6818\n",
      "Epoch 27/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.0314 - auc: 0.7855 - val_loss: 0.0335 - val_auc: 0.6833\n",
      "Epoch 28/40\n",
      "57/57 [==============================] - 9s 156ms/step - loss: 0.0311 - auc: 0.8094 - val_loss: 0.0336 - val_auc: 0.6781\n",
      "Epoch 29/40\n",
      "57/57 [==============================] - 8s 141ms/step - loss: 0.0316 - auc: 0.8081 - val_loss: 0.0336 - val_auc: 0.6812\n",
      "Epoch 30/40\n",
      "57/57 [==============================] - 8s 144ms/step - loss: 0.0318 - auc: 0.8120 - val_loss: 0.0336 - val_auc: 0.6794\n",
      "Epoch 31/40\n",
      "57/57 [==============================] - 10s 176ms/step - loss: 0.0315 - auc: 0.8134 - val_loss: 0.0335 - val_auc: 0.6840\n",
      "Epoch 32/40\n",
      "57/57 [==============================] - 9s 151ms/step - loss: 0.0316 - auc: 0.8175 - val_loss: 0.0335 - val_auc: 0.6888\n",
      "Epoch 33/40\n",
      "57/57 [==============================] - 11s 194ms/step - loss: 0.0311 - auc: 0.8236 - val_loss: 0.0336 - val_auc: 0.6848\n",
      "Epoch 34/40\n",
      "57/57 [==============================] - 9s 154ms/step - loss: 0.0310 - auc: 0.8149 - val_loss: 0.0335 - val_auc: 0.6867\n",
      "Epoch 35/40\n",
      "57/57 [==============================] - 9s 150ms/step - loss: 0.0312 - auc: 0.8162 - val_loss: 0.0335 - val_auc: 0.6905\n",
      "Epoch 36/40\n",
      "57/57 [==============================] - 9s 163ms/step - loss: 0.0322 - auc: 0.8223 - val_loss: 0.0334 - val_auc: 0.6920\n",
      "Epoch 37/40\n",
      "57/57 [==============================] - 10s 170ms/step - loss: 0.0316 - auc: 0.8131 - val_loss: 0.0334 - val_auc: 0.6996\n",
      "Epoch 38/40\n",
      "57/57 [==============================] - 9s 162ms/step - loss: 0.0307 - auc: 0.8201 - val_loss: 0.0334 - val_auc: 0.6971\n",
      "Epoch 39/40\n",
      "57/57 [==============================] - 10s 182ms/step - loss: 0.0312 - auc: 0.8129 - val_loss: 0.0334 - val_auc: 0.6897\n",
      "Epoch 40/40\n",
      "57/57 [==============================] - 11s 186ms/step - loss: 0.0303 - auc: 0.8262 - val_loss: 0.0335 - val_auc: 0.6881\n",
      "Fold ==> 1\n",
      "WARNING:tensorflow:`epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "Epoch 1/40\n",
      "57/57 [==============================] - 12s 153ms/step - loss: 0.7122 - auc: 0.5545 - val_loss: 0.6175 - val_auc: 0.5709\n",
      "Epoch 2/40\n",
      "57/57 [==============================] - 8s 149ms/step - loss: 0.6049 - auc: 0.5426 - val_loss: 0.4603 - val_auc: 0.5696\n",
      "Epoch 3/40\n",
      "57/57 [==============================] - 9s 163ms/step - loss: 0.4184 - auc: 0.5673 - val_loss: 0.2736 - val_auc: 0.5525\n",
      "Epoch 4/40\n",
      "57/57 [==============================] - 12s 203ms/step - loss: 0.2362 - auc: 0.5605 - val_loss: 0.1217 - val_auc: 0.5481\n",
      "Epoch 5/40\n",
      "57/57 [==============================] - 9s 160ms/step - loss: 0.1378 - auc: 0.5568 - val_loss: 0.0913 - val_auc: 0.5437\n",
      "Epoch 6/40\n",
      "57/57 [==============================] - 10s 168ms/step - loss: 0.0906 - auc: 0.5811 - val_loss: 0.0691 - val_auc: 0.5508\n",
      "Epoch 7/40\n",
      "57/57 [==============================] - 9s 152ms/step - loss: 0.0698 - auc: 0.6024 - val_loss: 0.0533 - val_auc: 0.5414\n",
      "Epoch 8/40\n",
      "57/57 [==============================] - 9s 157ms/step - loss: 0.0566 - auc: 0.6300 - val_loss: 0.0468 - val_auc: 0.5478\n",
      "Epoch 9/40\n",
      "57/57 [==============================] - 10s 184ms/step - loss: 0.0495 - auc: 0.6377 - val_loss: 0.0448 - val_auc: 0.5820\n",
      "Epoch 10/40\n",
      "57/57 [==============================] - 11s 196ms/step - loss: 0.0437 - auc: 0.6753 - val_loss: 0.0409 - val_auc: 0.5945\n",
      "Epoch 11/40\n",
      "57/57 [==============================] - 10s 175ms/step - loss: 0.0406 - auc: 0.6830 - val_loss: 0.0382 - val_auc: 0.5871\n",
      "Epoch 12/40\n",
      "57/57 [==============================] - 11s 185ms/step - loss: 0.0392 - auc: 0.7101 - val_loss: 0.0364 - val_auc: 0.5780\n",
      "Epoch 13/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.0370 - auc: 0.7077 - val_loss: 0.0368 - val_auc: 0.6172\n",
      "Epoch 14/40\n",
      "57/57 [==============================] - 11s 191ms/step - loss: 0.0356 - auc: 0.7447 - val_loss: 0.0346 - val_auc: 0.6136\n",
      "Epoch 15/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0345 - auc: 0.7672 - val_loss: 0.0352 - val_auc: 0.6145\n",
      "Epoch 16/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0340 - auc: 0.7606 - val_loss: 0.0344 - val_auc: 0.6398\n",
      "Epoch 17/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0327 - auc: 0.7490 - val_loss: 0.0339 - val_auc: 0.6436\n",
      "Epoch 18/40\n",
      "57/57 [==============================] - 11s 189ms/step - loss: 0.0329 - auc: 0.7795 - val_loss: 0.0339 - val_auc: 0.6465\n",
      "Epoch 19/40\n",
      "57/57 [==============================] - 10s 172ms/step - loss: 0.0323 - auc: 0.7529 - val_loss: 0.0341 - val_auc: 0.6598\n",
      "Epoch 20/40\n",
      "57/57 [==============================] - 10s 177ms/step - loss: 0.0318 - auc: 0.7956 - val_loss: 0.0341 - val_auc: 0.6547\n",
      "Epoch 21/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0321 - auc: 0.8001 - val_loss: 0.0338 - val_auc: 0.6608\n",
      "Epoch 22/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0302 - auc: 0.8126 - val_loss: 0.0336 - val_auc: 0.6700\n",
      "Epoch 23/40\n",
      "57/57 [==============================] - 11s 192ms/step - loss: 0.0313 - auc: 0.8080 - val_loss: 0.0334 - val_auc: 0.6725\n",
      "Epoch 24/40\n",
      "57/57 [==============================] - 11s 192ms/step - loss: 0.0317 - auc: 0.8169 - val_loss: 0.0332 - val_auc: 0.6847\n",
      "Epoch 25/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0310 - auc: 0.8141 - val_loss: 0.0332 - val_auc: 0.6843\n",
      "Epoch 26/40\n",
      "57/57 [==============================] - 11s 196ms/step - loss: 0.0303 - auc: 0.8244 - val_loss: 0.0332 - val_auc: 0.6867\n",
      "Epoch 27/40\n",
      "57/57 [==============================] - 11s 202ms/step - loss: 0.0307 - auc: 0.8025 - val_loss: 0.0332 - val_auc: 0.6916\n",
      "Epoch 28/40\n",
      "57/57 [==============================] - 11s 197ms/step - loss: 0.0287 - auc: 0.8276 - val_loss: 0.0332 - val_auc: 0.6896\n",
      "Epoch 29/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0323 - auc: 0.8084 - val_loss: 0.0332 - val_auc: 0.6866\n",
      "Epoch 30/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0298 - auc: 0.8182 - val_loss: 0.0332 - val_auc: 0.6868\n",
      "Epoch 31/40\n",
      "57/57 [==============================] - 10s 177ms/step - loss: 0.0301 - auc: 0.8213 - val_loss: 0.0332 - val_auc: 0.6859\n",
      "Epoch 32/40\n",
      "57/57 [==============================] - 10s 174ms/step - loss: 0.0314 - auc: 0.8148 - val_loss: 0.0332 - val_auc: 0.6862\n",
      "Epoch 33/40\n",
      "57/57 [==============================] - 8s 145ms/step - loss: 0.0292 - auc: 0.8363 - val_loss: 0.0332 - val_auc: 0.6874\n",
      "Epoch 34/40\n",
      "57/57 [==============================] - 9s 152ms/step - loss: 0.0306 - auc: 0.8177 - val_loss: 0.0332 - val_auc: 0.6869\n",
      "Epoch 35/40\n",
      "57/57 [==============================] - 9s 152ms/step - loss: 0.0304 - auc: 0.8191 - val_loss: 0.0332 - val_auc: 0.6867\n",
      "Epoch 36/40\n",
      "57/57 [==============================] - 9s 152ms/step - loss: 0.0302 - auc: 0.8085 - val_loss: 0.0331 - val_auc: 0.6876\n",
      "Epoch 37/40\n",
      "57/57 [==============================] - 8s 148ms/step - loss: 0.0314 - auc: 0.8256 - val_loss: 0.0331 - val_auc: 0.6888\n",
      "Epoch 38/40\n",
      "57/57 [==============================] - 9s 164ms/step - loss: 0.0312 - auc: 0.8226 - val_loss: 0.0330 - val_auc: 0.6896\n",
      "Epoch 39/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.0308 - auc: 0.8174 - val_loss: 0.0331 - val_auc: 0.6884\n",
      "Epoch 40/40\n",
      "57/57 [==============================] - 11s 185ms/step - loss: 0.0295 - auc: 0.8141 - val_loss: 0.0331 - val_auc: 0.6881\n",
      "Fold ==> 2\n",
      "WARNING:tensorflow:`epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "Epoch 1/40\n",
      "57/57 [==============================] - 15s 200ms/step - loss: 0.7141 - auc: 0.5044 - val_loss: 0.6142 - val_auc: 0.5399\n",
      "Epoch 2/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.6045 - auc: 0.5586 - val_loss: 0.4637 - val_auc: 0.5081\n",
      "Epoch 3/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.4168 - auc: 0.5580 - val_loss: 0.2600 - val_auc: 0.5163\n",
      "Epoch 4/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.2335 - auc: 0.5361 - val_loss: 0.1480 - val_auc: 0.5342\n",
      "Epoch 5/40\n",
      "57/57 [==============================] - 11s 194ms/step - loss: 0.1335 - auc: 0.5594 - val_loss: 0.0982 - val_auc: 0.5541\n",
      "Epoch 6/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0898 - auc: 0.5890 - val_loss: 0.0722 - val_auc: 0.5464\n",
      "Epoch 7/40\n",
      "57/57 [==============================] - 11s 185ms/step - loss: 0.0676 - auc: 0.6142 - val_loss: 0.0562 - val_auc: 0.5595\n",
      "Epoch 8/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0548 - auc: 0.6252 - val_loss: 0.0467 - val_auc: 0.5523\n",
      "Epoch 9/40\n",
      "57/57 [==============================] - 11s 186ms/step - loss: 0.0489 - auc: 0.6543 - val_loss: 0.0439 - val_auc: 0.5668\n",
      "Epoch 10/40\n",
      "57/57 [==============================] - 9s 151ms/step - loss: 0.0440 - auc: 0.6857 - val_loss: 0.0406 - val_auc: 0.5692\n",
      "Epoch 11/40\n",
      "57/57 [==============================] - 9s 166ms/step - loss: 0.0408 - auc: 0.6841 - val_loss: 0.0380 - val_auc: 0.5806\n",
      "Epoch 12/40\n",
      "57/57 [==============================] - 11s 196ms/step - loss: 0.0387 - auc: 0.7037 - val_loss: 0.0378 - val_auc: 0.5739\n",
      "Epoch 13/40\n",
      "57/57 [==============================] - 10s 166ms/step - loss: 0.0370 - auc: 0.7062 - val_loss: 0.0360 - val_auc: 0.5941\n",
      "Epoch 14/40\n",
      "57/57 [==============================] - 9s 154ms/step - loss: 0.0364 - auc: 0.7207 - val_loss: 0.0356 - val_auc: 0.6101\n",
      "Epoch 15/40\n",
      "57/57 [==============================] - 11s 189ms/step - loss: 0.0347 - auc: 0.7621 - val_loss: 0.0352 - val_auc: 0.5842\n",
      "Epoch 16/40\n",
      "57/57 [==============================] - 11s 191ms/step - loss: 0.0347 - auc: 0.7659 - val_loss: 0.0347 - val_auc: 0.6146\n",
      "Epoch 17/40\n",
      "57/57 [==============================] - 11s 186ms/step - loss: 0.0337 - auc: 0.7756 - val_loss: 0.0341 - val_auc: 0.6280\n",
      "Epoch 18/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0331 - auc: 0.7672 - val_loss: 0.0338 - val_auc: 0.6339\n",
      "Epoch 19/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0325 - auc: 0.7886 - val_loss: 0.0337 - val_auc: 0.6400\n",
      "Epoch 20/40\n",
      "57/57 [==============================] - 10s 180ms/step - loss: 0.0312 - auc: 0.7876 - val_loss: 0.0336 - val_auc: 0.6477\n",
      "Epoch 21/40\n",
      "57/57 [==============================] - 10s 180ms/step - loss: 0.0317 - auc: 0.8002 - val_loss: 0.0336 - val_auc: 0.6566\n",
      "Epoch 22/40\n",
      "57/57 [==============================] - 10s 179ms/step - loss: 0.0311 - auc: 0.7877 - val_loss: 0.0333 - val_auc: 0.6717\n",
      "Epoch 23/40\n",
      "57/57 [==============================] - 10s 180ms/step - loss: 0.0303 - auc: 0.8036 - val_loss: 0.0332 - val_auc: 0.6660\n",
      "Epoch 24/40\n",
      "57/57 [==============================] - 10s 179ms/step - loss: 0.0295 - auc: 0.8054 - val_loss: 0.0330 - val_auc: 0.6853\n",
      "Epoch 25/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0309 - auc: 0.8307 - val_loss: 0.0330 - val_auc: 0.7102\n",
      "Epoch 26/40\n",
      "57/57 [==============================] - 10s 179ms/step - loss: 0.0302 - auc: 0.8399 - val_loss: 0.0330 - val_auc: 0.6727\n",
      "Epoch 27/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0295 - auc: 0.8191 - val_loss: 0.0328 - val_auc: 0.7039\n",
      "Epoch 28/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0302 - auc: 0.8338 - val_loss: 0.0327 - val_auc: 0.7184\n",
      "Epoch 29/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0301 - auc: 0.8332 - val_loss: 0.0324 - val_auc: 0.7393\n",
      "Epoch 30/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0285 - auc: 0.8366 - val_loss: 0.0326 - val_auc: 0.7179\n",
      "Epoch 31/40\n",
      "57/57 [==============================] - 10s 177ms/step - loss: 0.0289 - auc: 0.8513 - val_loss: 0.0324 - val_auc: 0.7249\n",
      "Epoch 32/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0276 - auc: 0.8555 - val_loss: 0.0324 - val_auc: 0.7235\n",
      "Epoch 33/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0273 - auc: 0.8741 - val_loss: 0.0323 - val_auc: 0.7315\n",
      "Epoch 34/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0282 - auc: 0.8675 - val_loss: 0.0322 - val_auc: 0.7392\n",
      "Epoch 35/40\n",
      "57/57 [==============================] - 10s 177ms/step - loss: 0.0278 - auc: 0.8478 - val_loss: 0.0319 - val_auc: 0.7467\n",
      "Epoch 36/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0276 - auc: 0.8676 - val_loss: 0.0320 - val_auc: 0.7490\n",
      "Epoch 37/40\n",
      "57/57 [==============================] - 10s 179ms/step - loss: 0.0277 - auc: 0.8740 - val_loss: 0.0319 - val_auc: 0.7513\n",
      "Epoch 38/40\n",
      "57/57 [==============================] - 10s 181ms/step - loss: 0.0273 - auc: 0.8603 - val_loss: 0.0319 - val_auc: 0.7488\n",
      "Epoch 39/40\n",
      "57/57 [==============================] - 11s 193ms/step - loss: 0.0265 - auc: 0.8635 - val_loss: 0.0320 - val_auc: 0.7407\n",
      "Epoch 40/40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57/57 [==============================] - 13s 222ms/step - loss: 0.0272 - auc: 0.8699 - val_loss: 0.0319 - val_auc: 0.7493\n",
      "Fold ==> 3\n",
      "WARNING:tensorflow:`epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "Epoch 1/40\n",
      "57/57 [==============================] - 19s 220ms/step - loss: 0.7119 - auc: 0.5350 - val_loss: 0.7188 - val_auc: 0.6083\n",
      "Epoch 2/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.6020 - auc: 0.5503 - val_loss: 0.4705 - val_auc: 0.6196\n",
      "Epoch 3/40\n",
      "57/57 [==============================] - 11s 197ms/step - loss: 0.4129 - auc: 0.5473 - val_loss: 0.2684 - val_auc: 0.6176\n",
      "Epoch 4/40\n",
      "57/57 [==============================] - 12s 218ms/step - loss: 0.2322 - auc: 0.5640 - val_loss: 0.1431 - val_auc: 0.6212\n",
      "Epoch 5/40\n",
      "57/57 [==============================] - 11s 195ms/step - loss: 0.1347 - auc: 0.5817 - val_loss: 0.0955 - val_auc: 0.6086\n",
      "Epoch 6/40\n",
      "57/57 [==============================] - 12s 212ms/step - loss: 0.0897 - auc: 0.5634 - val_loss: 0.0700 - val_auc: 0.6474\n",
      "Epoch 7/40\n",
      "57/57 [==============================] - 11s 197ms/step - loss: 0.0678 - auc: 0.6035 - val_loss: 0.0546 - val_auc: 0.6612\n",
      "Epoch 8/40\n",
      "57/57 [==============================] - 13s 222ms/step - loss: 0.0555 - auc: 0.6257 - val_loss: 0.0471 - val_auc: 0.6956\n",
      "Epoch 9/40\n",
      "57/57 [==============================] - 12s 211ms/step - loss: 0.0484 - auc: 0.6687 - val_loss: 0.0425 - val_auc: 0.6896\n",
      "Epoch 10/40\n",
      "57/57 [==============================] - 11s 191ms/step - loss: 0.0425 - auc: 0.6984 - val_loss: 0.0395 - val_auc: 0.7117\n",
      "Epoch 11/40\n",
      "57/57 [==============================] - 11s 193ms/step - loss: 0.0396 - auc: 0.6893 - val_loss: 0.0377 - val_auc: 0.7101\n",
      "Epoch 12/40\n",
      "57/57 [==============================] - 11s 192ms/step - loss: 0.0386 - auc: 0.7167 - val_loss: 0.0363 - val_auc: 0.6963\n",
      "Epoch 13/40\n",
      "57/57 [==============================] - 12s 213ms/step - loss: 0.0376 - auc: 0.7138 - val_loss: 0.0349 - val_auc: 0.7143\n",
      "Epoch 14/40\n",
      "57/57 [==============================] - 13s 233ms/step - loss: 0.0360 - auc: 0.7408 - val_loss: 0.0342 - val_auc: 0.7343\n",
      "Epoch 15/40\n",
      "57/57 [==============================] - 11s 193ms/step - loss: 0.0336 - auc: 0.7385 - val_loss: 0.0333 - val_auc: 0.7225\n",
      "Epoch 16/40\n",
      "57/57 [==============================] - 11s 191ms/step - loss: 0.0331 - auc: 0.7712 - val_loss: 0.0327 - val_auc: 0.7504\n",
      "Epoch 17/40\n",
      "57/57 [==============================] - 11s 193ms/step - loss: 0.0338 - auc: 0.7790 - val_loss: 0.0325 - val_auc: 0.7254\n",
      "Epoch 18/40\n",
      "57/57 [==============================] - 11s 196ms/step - loss: 0.0325 - auc: 0.7891 - val_loss: 0.0319 - val_auc: 0.7640\n",
      "Epoch 19/40\n",
      "57/57 [==============================] - 12s 214ms/step - loss: 0.0318 - auc: 0.7869 - val_loss: 0.0317 - val_auc: 0.7745\n",
      "Epoch 20/40\n",
      "57/57 [==============================] - 11s 197ms/step - loss: 0.0323 - auc: 0.7992 - val_loss: 0.0312 - val_auc: 0.7925\n",
      "Epoch 21/40\n",
      "57/57 [==============================] - 11s 198ms/step - loss: 0.0308 - auc: 0.8243 - val_loss: 0.0318 - val_auc: 0.7492:\n",
      "Epoch 22/40\n",
      "57/57 [==============================] - 11s 196ms/step - loss: 0.0299 - auc: 0.8258 - val_loss: 0.0308 - val_auc: 0.8121\n",
      "Epoch 23/40\n",
      "57/57 [==============================] - 13s 228ms/step - loss: 0.0311 - auc: 0.8204 - val_loss: 0.0304 - val_auc: 0.8084\n",
      "Epoch 24/40\n",
      "57/57 [==============================] - 12s 209ms/step - loss: 0.0310 - auc: 0.8297 - val_loss: 0.0306 - val_auc: 0.7911\n",
      "Epoch 25/40\n",
      "57/57 [==============================] - 12s 205ms/step - loss: 0.0287 - auc: 0.8328 - val_loss: 0.0312 - val_auc: 0.8019\n",
      "Epoch 26/40\n",
      "57/57 [==============================] - 12s 211ms/step - loss: 0.0295 - auc: 0.8227 - val_loss: 0.0305 - val_auc: 0.7990\n",
      "Epoch 27/40\n",
      "57/57 [==============================] - 10s 176ms/step - loss: 0.0290 - auc: 0.8593 - val_loss: 0.0299 - val_auc: 0.8224\n",
      "Epoch 28/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.0280 - auc: 0.8518 - val_loss: 0.0298 - val_auc: 0.8236\n",
      "Epoch 29/40\n",
      "57/57 [==============================] - 11s 186ms/step - loss: 0.0279 - auc: 0.8502 - val_loss: 0.0296 - val_auc: 0.8242\n",
      "Epoch 30/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0285 - auc: 0.8536 - val_loss: 0.0296 - val_auc: 0.8272\n",
      "Epoch 31/40\n",
      "57/57 [==============================] - 10s 184ms/step - loss: 0.0289 - auc: 0.8696 - val_loss: 0.0295 - val_auc: 0.8243\n",
      "Epoch 32/40\n",
      "57/57 [==============================] - 10s 183ms/step - loss: 0.0278 - auc: 0.8605 - val_loss: 0.0293 - val_auc: 0.8315\n",
      "Epoch 33/40\n",
      "57/57 [==============================] - 10s 182ms/step - loss: 0.0275 - auc: 0.8695 - val_loss: 0.0292 - val_auc: 0.8276\n",
      "Epoch 34/40\n",
      "57/57 [==============================] - 10s 182ms/step - loss: 0.0294 - auc: 0.8594 - val_loss: 0.0291 - val_auc: 0.8224\n",
      "Epoch 35/40\n",
      "57/57 [==============================] - 10s 182ms/step - loss: 0.0285 - auc: 0.8785 - val_loss: 0.0291 - val_auc: 0.8286\n",
      "Epoch 36/40\n",
      "57/57 [==============================] - 10s 182ms/step - loss: 0.0277 - auc: 0.8605 - val_loss: 0.0290 - val_auc: 0.8222\n",
      "Epoch 37/40\n",
      "57/57 [==============================] - 10s 181ms/step - loss: 0.0276 - auc: 0.8594 - val_loss: 0.0291 - val_auc: 0.8281\n",
      "Epoch 38/40\n",
      "57/57 [==============================] - 10s 182ms/step - loss: 0.0279 - auc: 0.8696 - val_loss: 0.0292 - val_auc: 0.8273\n",
      "Epoch 39/40\n",
      "57/57 [==============================] - 10s 181ms/step - loss: 0.0285 - auc: 0.8695 - val_loss: 0.0290 - val_auc: 0.8238\n",
      "Epoch 40/40\n",
      "57/57 [==============================] - 10s 180ms/step - loss: 0.0282 - auc: 0.8731 - val_loss: 0.0290 - val_auc: 0.8284\n",
      "Fold ==> 4\n",
      "WARNING:tensorflow:`epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "Epoch 1/40\n",
      "57/57 [==============================] - 14s 187ms/step - loss: 0.7199 - auc: 0.5043 - val_loss: 0.6202 - val_auc: 0.5795\n",
      "Epoch 2/40\n",
      "57/57 [==============================] - 10s 177ms/step - loss: 0.6108 - auc: 0.5485 - val_loss: 0.4472 - val_auc: 0.5758\n",
      "Epoch 3/40\n",
      "57/57 [==============================] - 10s 179ms/step - loss: 0.4191 - auc: 0.5581 - val_loss: 0.2651 - val_auc: 0.5664\n",
      "Epoch 4/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.2375 - auc: 0.5748 - val_loss: 0.1366 - val_auc: 0.5453\n",
      "Epoch 5/40\n",
      "57/57 [==============================] - 10s 179ms/step - loss: 0.1378 - auc: 0.5853 - val_loss: 0.0933 - val_auc: 0.5549\n",
      "Epoch 6/40\n",
      "57/57 [==============================] - 10s 178ms/step - loss: 0.0916 - auc: 0.5893 - val_loss: 0.0693 - val_auc: 0.5726\n",
      "Epoch 7/40\n",
      "57/57 [==============================] - 10s 184ms/step - loss: 0.0703 - auc: 0.6125 - val_loss: 0.0557 - val_auc: 0.5662\n",
      "Epoch 8/40\n",
      "57/57 [==============================] - 12s 210ms/step - loss: 0.0565 - auc: 0.6442 - val_loss: 0.0471 - val_auc: 0.5849\n",
      "Epoch 9/40\n",
      "57/57 [==============================] - 11s 186ms/step - loss: 0.0496 - auc: 0.6637 - val_loss: 0.0446 - val_auc: 0.6047\n",
      "Epoch 10/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0451 - auc: 0.6712 - val_loss: 0.0432 - val_auc: 0.6091\n",
      "Epoch 11/40\n",
      "57/57 [==============================] - 11s 196ms/step - loss: 0.0417 - auc: 0.6772 - val_loss: 0.0388 - val_auc: 0.6192\n",
      "Epoch 12/40\n",
      "57/57 [==============================] - 11s 191ms/step - loss: 0.0401 - auc: 0.6856 - val_loss: 0.0368 - val_auc: 0.6050\n",
      "Epoch 13/40\n",
      "57/57 [==============================] - 11s 199ms/step - loss: 0.0377 - auc: 0.7123 - val_loss: 0.0363 - val_auc: 0.6203\n",
      "Epoch 14/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0351 - auc: 0.7249 - val_loss: 0.0353 - val_auc: 0.6174350 - auc\n",
      "Epoch 15/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.0356 - auc: 0.7339 - val_loss: 0.0358 - val_auc: 0.6299\n",
      "Epoch 16/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0335 - auc: 0.7552 - val_loss: 0.0344 - val_auc: 0.6364 3s - loss: 0\n",
      "Epoch 17/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0329 - auc: 0.7664 - val_loss: 0.0345 - val_auc: 0.6410\n",
      "Epoch 18/40\n",
      "57/57 [==============================] - 13s 223ms/step - loss: 0.0342 - auc: 0.7828 - val_loss: 0.0338 - val_auc: 0.6361\n",
      "Epoch 19/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0321 - auc: 0.7639 - val_loss: 0.0338 - val_auc: 0.6554\n",
      "Epoch 20/40\n",
      "57/57 [==============================] - 11s 187ms/step - loss: 0.0323 - auc: 0.7812 - val_loss: 0.0335 - val_auc: 0.6305\n",
      "Epoch 21/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0320 - auc: 0.7920 - val_loss: 0.0338 - val_auc: 0.6796\n",
      "Epoch 22/40\n",
      "57/57 [==============================] - 11s 201ms/step - loss: 0.0304 - auc: 0.8031 - val_loss: 0.0332 - val_auc: 0.6740\n",
      "Epoch 23/40\n",
      "57/57 [==============================] - 11s 192ms/step - loss: 0.0314 - auc: 0.8135 - val_loss: 0.0331 - val_auc: 0.6791\n",
      "Epoch 24/40\n",
      "57/57 [==============================] - 11s 188ms/step - loss: 0.0308 - auc: 0.8176 - val_loss: 0.0329 - val_auc: 0.6888\n",
      "Epoch 25/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0292 - auc: 0.8414 - val_loss: 0.0330 - val_auc: 0.6789 \n",
      "Epoch 26/40\n",
      "57/57 [==============================] - 11s 198ms/step - loss: 0.0306 - auc: 0.8217 - val_loss: 0.0326 - val_auc: 0.7158\n",
      "Epoch 27/40\n",
      "57/57 [==============================] - 12s 204ms/step - loss: 0.0294 - auc: 0.8402 - val_loss: 0.0327 - val_auc: 0.6977\n",
      "Epoch 28/40\n",
      "57/57 [==============================] - 11s 195ms/step - loss: 0.0283 - auc: 0.8533 - val_loss: 0.0337 - val_auc: 0.6443\n",
      "Epoch 29/40\n",
      "57/57 [==============================] - 11s 194ms/step - loss: 0.0288 - auc: 0.8332 - val_loss: 0.0330 - val_auc: 0.6833\n",
      "Epoch 30/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0294 - auc: 0.8612 - val_loss: 0.0327 - val_auc: 0.7127\n",
      "Epoch 31/40\n",
      "57/57 [==============================] - 11s 191ms/step - loss: 0.0285 - auc: 0.8669 - val_loss: 0.0325 - val_auc: 0.7211\n",
      "Epoch 32/40\n",
      "57/57 [==============================] - 12s 219ms/step - loss: 0.0277 - auc: 0.8666 - val_loss: 0.0325 - val_auc: 0.7225\n",
      "Epoch 33/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0291 - auc: 0.8766 - val_loss: 0.0323 - val_auc: 0.7322\n",
      "Epoch 34/40\n",
      "57/57 [==============================] - 11s 189ms/step - loss: 0.0270 - auc: 0.8676 - val_loss: 0.0322 - val_auc: 0.7456\n",
      "Epoch 35/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0281 - auc: 0.8598 - val_loss: 0.0320 - val_auc: 0.7420\n",
      "Epoch 36/40\n",
      "57/57 [==============================] - 12s 203ms/step - loss: 0.0281 - auc: 0.8729 - val_loss: 0.0321 - val_auc: 0.7459\n",
      "Epoch 37/40\n",
      "57/57 [==============================] - 14s 239ms/step - loss: 0.0277 - auc: 0.8711 - val_loss: 0.0322 - val_auc: 0.7424\n",
      "Epoch 38/40\n",
      "57/57 [==============================] - 12s 214ms/step - loss: 0.0257 - auc: 0.8703 - val_loss: 0.0321 - val_auc: 0.7479\n",
      "Epoch 39/40\n",
      "57/57 [==============================] - 11s 190ms/step - loss: 0.0281 - auc: 0.8674 - val_loss: 0.0320 - val_auc: 0.7479\n",
      "Epoch 40/40\n",
      "57/57 [==============================] - 11s 194ms/step - loss: 0.0280 - auc: 0.8633 - val_loss: 0.0320 - val_auc: 0.7488\n"
     ]
    }
   ],
   "source": [
    "N_SPLITS = 5\n",
    "RNG_SEED = 42\n",
    "EPOCHS = 40\n",
    "BATCH_SIZE=128\n",
    "\n",
    "ytrain = np.zeros((TRAIN.shape[0],len(tar_col)))\n",
    "ytest = np.zeros((TEST.shape[0],len(tar_col)))\n",
    "\n",
    "for n, (tr, te) in enumerate(\n",
    "    IterativeStratification(n_splits=N_SPLITS, random_state=RNG_SEED)\n",
    "    .split(y_train_all_classes[tar_col], y_train_all_classes[tar_col])):\n",
    "    \n",
    "    model = build_model()\n",
    "    print(f\"Fold ==> {n}\")\n",
    "    reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3,\n",
    "                                       verbose=0, epsilon=1e-4, mode='min')\n",
    "    model.fit(TRAIN[tr,:],\n",
    "              TRAIN_TARGETS[tr,:],\n",
    "              validation_data=(TRAIN[te,:], TRAIN_TARGETS[te,:]),\n",
    "              epochs=EPOCHS, batch_size=BATCH_SIZE,\n",
    "              callbacks=[reduce_lr_loss], verbose=1,\n",
    "             )\n",
    "    \n",
    "    ytrain[te,:] += model.predict(TRAIN[te,:])\n",
    "    ytest +=  model.predict(TEST)/N_SPLITS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('saved_model/my_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean of AUC scores = 0.7640451897590913\n"
     ]
    }
   ],
   "source": [
    "AUC_scores = []\n",
    "for n  in range(len(tar_col)): \n",
    "    AUC_scores.append(roc_auc_score(TRAIN_TARGETS[:,n], ytrain[:,n]))\n",
    "    #print( f'Target{n} AUC', roc_auc_score(TRAIN_TARGETS[:,n], ytrain[:,n]) )\n",
    "print(f'mean of AUC scores = {np.mean(AUC_scores)}') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>recording_id</th>\n",
       "      <th>s0</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>s6</th>\n",
       "      <th>s7</th>\n",
       "      <th>s8</th>\n",
       "      <th>...</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "      <th>s22</th>\n",
       "      <th>s23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000316da7</td>\n",
       "      <td>0.006161</td>\n",
       "      <td>0.005675</td>\n",
       "      <td>0.004037</td>\n",
       "      <td>0.005231</td>\n",
       "      <td>0.004419</td>\n",
       "      <td>0.006131</td>\n",
       "      <td>0.005999</td>\n",
       "      <td>0.006391</td>\n",
       "      <td>0.005704</td>\n",
       "      <td>...</td>\n",
       "      <td>0.009689</td>\n",
       "      <td>0.005181</td>\n",
       "      <td>0.003585</td>\n",
       "      <td>0.006545</td>\n",
       "      <td>0.006226</td>\n",
       "      <td>0.005239</td>\n",
       "      <td>0.007672</td>\n",
       "      <td>0.006276</td>\n",
       "      <td>0.009956</td>\n",
       "      <td>0.007983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>003bc2cb2</td>\n",
       "      <td>0.003939</td>\n",
       "      <td>0.011657</td>\n",
       "      <td>0.003939</td>\n",
       "      <td>0.008894</td>\n",
       "      <td>0.003298</td>\n",
       "      <td>0.001888</td>\n",
       "      <td>0.002304</td>\n",
       "      <td>0.005089</td>\n",
       "      <td>0.003289</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005366</td>\n",
       "      <td>0.007275</td>\n",
       "      <td>0.091355</td>\n",
       "      <td>0.021637</td>\n",
       "      <td>0.003819</td>\n",
       "      <td>0.005712</td>\n",
       "      <td>0.007750</td>\n",
       "      <td>0.003282</td>\n",
       "      <td>0.003248</td>\n",
       "      <td>0.002667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0061c037e</td>\n",
       "      <td>0.005391</td>\n",
       "      <td>0.005367</td>\n",
       "      <td>0.004225</td>\n",
       "      <td>0.004925</td>\n",
       "      <td>0.003714</td>\n",
       "      <td>0.006915</td>\n",
       "      <td>0.008413</td>\n",
       "      <td>0.006760</td>\n",
       "      <td>0.004611</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007868</td>\n",
       "      <td>0.005261</td>\n",
       "      <td>0.003647</td>\n",
       "      <td>0.008345</td>\n",
       "      <td>0.005293</td>\n",
       "      <td>0.006108</td>\n",
       "      <td>0.009097</td>\n",
       "      <td>0.004920</td>\n",
       "      <td>0.011893</td>\n",
       "      <td>0.008345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>010eb14d3</td>\n",
       "      <td>0.014188</td>\n",
       "      <td>0.004181</td>\n",
       "      <td>0.002437</td>\n",
       "      <td>0.004052</td>\n",
       "      <td>0.003010</td>\n",
       "      <td>0.003374</td>\n",
       "      <td>0.002041</td>\n",
       "      <td>0.003769</td>\n",
       "      <td>0.005537</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008295</td>\n",
       "      <td>0.003531</td>\n",
       "      <td>0.002725</td>\n",
       "      <td>0.003229</td>\n",
       "      <td>0.005944</td>\n",
       "      <td>0.003025</td>\n",
       "      <td>0.004287</td>\n",
       "      <td>0.003747</td>\n",
       "      <td>0.003642</td>\n",
       "      <td>0.004799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>011318064</td>\n",
       "      <td>0.008300</td>\n",
       "      <td>0.004755</td>\n",
       "      <td>0.003386</td>\n",
       "      <td>0.004891</td>\n",
       "      <td>0.004372</td>\n",
       "      <td>0.005797</td>\n",
       "      <td>0.003434</td>\n",
       "      <td>0.004998</td>\n",
       "      <td>0.005400</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013242</td>\n",
       "      <td>0.004014</td>\n",
       "      <td>0.002880</td>\n",
       "      <td>0.004044</td>\n",
       "      <td>0.007807</td>\n",
       "      <td>0.003762</td>\n",
       "      <td>0.005487</td>\n",
       "      <td>0.005746</td>\n",
       "      <td>0.006298</td>\n",
       "      <td>0.008658</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  recording_id        s0        s1        s2        s3        s4        s5  \\\n",
       "0    000316da7  0.006161  0.005675  0.004037  0.005231  0.004419  0.006131   \n",
       "1    003bc2cb2  0.003939  0.011657  0.003939  0.008894  0.003298  0.001888   \n",
       "2    0061c037e  0.005391  0.005367  0.004225  0.004925  0.003714  0.006915   \n",
       "3    010eb14d3  0.014188  0.004181  0.002437  0.004052  0.003010  0.003374   \n",
       "4    011318064  0.008300  0.004755  0.003386  0.004891  0.004372  0.005797   \n",
       "\n",
       "         s6        s7        s8  ...       s14       s15       s16       s17  \\\n",
       "0  0.005999  0.006391  0.005704  ...  0.009689  0.005181  0.003585  0.006545   \n",
       "1  0.002304  0.005089  0.003289  ...  0.005366  0.007275  0.091355  0.021637   \n",
       "2  0.008413  0.006760  0.004611  ...  0.007868  0.005261  0.003647  0.008345   \n",
       "3  0.002041  0.003769  0.005537  ...  0.008295  0.003531  0.002725  0.003229   \n",
       "4  0.003434  0.004998  0.005400  ...  0.013242  0.004014  0.002880  0.004044   \n",
       "\n",
       "        s18       s19       s20       s21       s22       s23  \n",
       "0  0.006226  0.005239  0.007672  0.006276  0.009956  0.007983  \n",
       "1  0.003819  0.005712  0.007750  0.003282  0.003248  0.002667  \n",
       "2  0.005293  0.006108  0.009097  0.004920  0.011893  0.008345  \n",
       "3  0.005944  0.003025  0.004287  0.003747  0.003642  0.004799  \n",
       "4  0.007807  0.003762  0.005487  0.005746  0.006298  0.008658  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission[tar_col] = ytest\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission.to_csv('savedata/TF1/submission-tf-fft-1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1992, 25)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}