ResNet-VGG-Inception-4Piece-PCA-XGBoost-Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fd1efdf1",
   "metadata": {},
   "source": [
    "### Importing Necessary Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "0938de48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.preprocessing.image import load_img, img_to_array\n",
    "\n",
    "from tensorflow.keras.layers import Input, Flatten, Dense, Dropout, GlobalAvgPool2D, BatchNormalization\n",
    "# from tensorflow.keras.applications import EfficientNetV2L\n",
    "from tensorflow.keras.regularizers import l2\n",
    "\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "\n",
    "from sklearn.metrics import precision_recall_fscore_support as score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c954b162",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68302673",
   "metadata": {},
   "source": [
    "A detailed discussion and visualization of the data can be seen in [here](Data-Overview.ipynb)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "75b06a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.read_csv(\"./Data/train_labels.csv\")\n",
    "images = ['./Data/train/train/'+str(name)+'.tif' for name in labels['id']]\n",
    "labels['images'] = images\n",
    "labels = labels[['id', 'images', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fd2fe082",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>./Data/train/train/0.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>./Data/train/train/1.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>./Data/train/train/2.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>./Data/train/train/3.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>./Data/train/train/4.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15995</th>\n",
       "      <td>15995</td>\n",
       "      <td>./Data/train/train/15995.tif</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15996</th>\n",
       "      <td>15996</td>\n",
       "      <td>./Data/train/train/15996.tif</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15997</th>\n",
       "      <td>15997</td>\n",
       "      <td>./Data/train/train/15997.tif</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15998</th>\n",
       "      <td>15998</td>\n",
       "      <td>./Data/train/train/15998.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15999</th>\n",
       "      <td>15999</td>\n",
       "      <td>./Data/train/train/15999.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                        images  label\n",
       "0          0      ./Data/train/train/0.tif      1\n",
       "1          1      ./Data/train/train/1.tif     13\n",
       "2          2      ./Data/train/train/2.tif     13\n",
       "3          3      ./Data/train/train/3.tif     14\n",
       "4          4      ./Data/train/train/4.tif      6\n",
       "...      ...                           ...    ...\n",
       "15995  15995  ./Data/train/train/15995.tif      2\n",
       "15996  15996  ./Data/train/train/15996.tif     15\n",
       "15997  15997  ./Data/train/train/15997.tif      3\n",
       "15998  15998  ./Data/train/train/15998.tif      9\n",
       "15999  15999  ./Data/train/train/15999.tif      9\n",
       "\n",
       "[16000 rows x 3 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "276e2059",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = pd.read_csv('./Data/train_labels_1.csv')\n",
    "test_labels = pd.read_csv('./Data/test_labels_1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2aea9ea9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>./Data/train/train/0.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>./Data/train/train/1.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>./Data/train/train/2.tif</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>./Data/train/train/3.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>./Data/train/train/4.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15195</th>\n",
       "      <td>15995</td>\n",
       "      <td>./Data/train/train/15995.tif</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15196</th>\n",
       "      <td>15996</td>\n",
       "      <td>./Data/train/train/15996.tif</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15197</th>\n",
       "      <td>15997</td>\n",
       "      <td>./Data/train/train/15997.tif</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15198</th>\n",
       "      <td>15998</td>\n",
       "      <td>./Data/train/train/15998.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15199</th>\n",
       "      <td>15999</td>\n",
       "      <td>./Data/train/train/15999.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15200 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                        images  label\n",
       "0          0      ./Data/train/train/0.tif      1\n",
       "1          1      ./Data/train/train/1.tif     13\n",
       "2          2      ./Data/train/train/2.tif     13\n",
       "3          3      ./Data/train/train/3.tif     14\n",
       "4          4      ./Data/train/train/4.tif      6\n",
       "...      ...                           ...    ...\n",
       "15195  15995  ./Data/train/train/15995.tif      2\n",
       "15196  15996  ./Data/train/train/15996.tif     15\n",
       "15197  15997  ./Data/train/train/15997.tif      3\n",
       "15198  15998  ./Data/train/train/15998.tif      9\n",
       "15199  15999  ./Data/train/train/15999.tif      9\n",
       "\n",
       "[15200 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b9ea19c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18</td>\n",
       "      <td>./Data/train/train/18.tif</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>./Data/train/train/25.tif</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>./Data/train/train/33.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41</td>\n",
       "      <td>./Data/train/train/41.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65</td>\n",
       "      <td>./Data/train/train/65.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>15816</td>\n",
       "      <td>./Data/train/train/15816.tif</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>15824</td>\n",
       "      <td>./Data/train/train/15824.tif</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>15832</td>\n",
       "      <td>./Data/train/train/15832.tif</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>15863</td>\n",
       "      <td>./Data/train/train/15863.tif</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>15994</td>\n",
       "      <td>./Data/train/train/15994.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>800 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                        images  label\n",
       "0       18     ./Data/train/train/18.tif     11\n",
       "1       25     ./Data/train/train/25.tif      8\n",
       "2       33     ./Data/train/train/33.tif      6\n",
       "3       41     ./Data/train/train/41.tif      1\n",
       "4       65     ./Data/train/train/65.tif     14\n",
       "..     ...                           ...    ...\n",
       "795  15816  ./Data/train/train/15816.tif      2\n",
       "796  15824  ./Data/train/train/15824.tif      2\n",
       "797  15832  ./Data/train/train/15832.tif     11\n",
       "798  15863  ./Data/train/train/15863.tif     15\n",
       "799  15994  ./Data/train/train/15994.tif      6\n",
       "\n",
       "[800 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "29190fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_labels = list(set(labels['label']))\n",
    "class_names = [\n",
    "    'letter', 'form', 'email', 'handwritten', 'advertisement', 'scientific report', 'scientific publication',\n",
    "    'specification', 'file folder', 'news article', 'budget', 'invoice', 'presentation', 'questionnaire', 'resume',\n",
    "    'memo'\n",
    "]\n",
    "label_names = pd.DataFrame({\n",
    "    'labels': class_labels,\n",
    "    'names': class_names\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2e251c1",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85b2ccd1",
   "metadata": {},
   "source": [
    "We have already generated the Extracted Feature Vectors from the ResNet Model now we train using that, so we load those feature vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "37a36e2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 2048)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_resnet = np.load('./Data/ResNet-4P-train-precompute/whole_train_data_precomp.npy')\n",
    "fine_tune_data_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a77e3e21",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_COMPS_1 = 1300 # 1600\n",
    "N_COMPS_2 = 300 # 400\n",
    "N_COMPS_3 = 900 # 1000\n",
    "random_state = 43"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3fce6fd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pcas_resnet = [PCA(N_COMPS_1, random_state=random_state) for _ in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6ca01f81",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    pcas_resnet[i].fit(fine_tune_data_resnet[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "349f0b3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "fine_tune_data_train_red_resnet = np.zeros((fine_tune_data_resnet.shape[0], 5, N_COMPS_1))\n",
    "for i in range(5):\n",
    "    fine_tune_data_train_red_resnet[:,i,:] = pcas_resnet[i].transform(fine_tune_data_resnet[:,i,:])\n",
    "fine_tune_data_resnet = fine_tune_data_train_red_resnet\n",
    "del fine_tune_data_train_red_resnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4882f218",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 1300)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4cc9fe34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 512)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_vgg = np.load('./Data/VGG16-4P-train-precompute/whole_train_data_precomp.npy')\n",
    "fine_tune_data_vgg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a900a6a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pcas_vgg = [PCA(N_COMPS_2, random_state=random_state) for _ in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "59dbb451",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    pcas_vgg[i].fit(fine_tune_data_vgg[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d38ff84f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 300)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_train_red_vgg = np.zeros((fine_tune_data_vgg.shape[0], 5, N_COMPS_2))\n",
    "for i in range(5):\n",
    "    fine_tune_data_train_red_vgg[:,i,:] = pcas_vgg[i].transform(fine_tune_data_vgg[:,i,:])\n",
    "fine_tune_data_vgg = fine_tune_data_train_red_vgg\n",
    "del fine_tune_data_train_red_vgg\n",
    "fine_tune_data_vgg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1178e7cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 1536)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_inception_resnet = np.load('./Data/Inception-ResNet-4P-train-precompute/whole_train_data_precomp.npy')\n",
    "fine_tune_data_inception_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "63c8d039",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pcas_inception_resnet = [PCA(N_COMPS_3, random_state=random_state) for _ in range(5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a66a0788",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    pcas_inception_resnet[i].fit(fine_tune_data_inception_resnet[:,i,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2553c06a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 900)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_train_red_inception_resnet = np.zeros((fine_tune_data_inception_resnet.shape[0], 5, N_COMPS_3))\n",
    "for i in range(5):\n",
    "    fine_tune_data_train_red_inception_resnet[:,i,:] = pcas_inception_resnet[i].transform(fine_tune_data_inception_resnet[:,i,:])\n",
    "fine_tune_data_inception_resnet = fine_tune_data_train_red_inception_resnet\n",
    "del fine_tune_data_train_red_inception_resnet\n",
    "fine_tune_data_inception_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6e4e8b18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000, 5, 2500)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data = np.concatenate([fine_tune_data_resnet, fine_tune_data_vgg, fine_tune_data_inception_resnet], axis=-1)\n",
    "fine_tune_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "568ffcb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16000,)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_labels = labels['label'].to_numpy()\n",
    "fine_tune_labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "937e1e86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15200, 5, 2500)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_train = fine_tune_data[list(train_labels['id'])]\n",
    "fine_tune_data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "af08c830",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15200,)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_labels_train = train_labels['label'].to_numpy()\n",
    "fine_tune_labels_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c00f0b32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800, 5, 2500)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_test = fine_tune_data[list(test_labels['id'])]\n",
    "fine_tune_data_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "16f4bb93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(800,)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_labels_test = test_labels['label'].to_numpy()\n",
    "fine_tune_labels_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64e65496",
   "metadata": {},
   "source": [
    "Loading the CNN based trained model and generating data for XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "96a2bdfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ftmodel():\n",
    "    inputs = Input(shape=(5, N_COMPS_1 + N_COMPS_2 + N_COMPS_3))\n",
    "    x = Flatten()(inputs)\n",
    "    x = BatchNormalization()(x)\n",
    "\n",
    "    top_dropout_rate = 0.5\n",
    "    # x = Dropout(top_dropout_rate, name='top_dropout_1')(x)\n",
    "    # x = Dense(1024, name='top_dense_1', kernel_initializer=\"he_normal\", kernel_regularizer=l2(5e-5), activation='relu')(x)\n",
    "    # x = BatchNormalization()(x)\n",
    "    x = Dropout(top_dropout_rate, name='top_dropout_2')(x)\n",
    "    x = Dense(512, name='top_dense_2', kernel_initializer=\"he_normal\", kernel_regularizer=l2(5e-5), activation='relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dropout(top_dropout_rate, name='top_dropout_3')(x)\n",
    "    # x = Dense(256, name='top_dense_3', kernel_initializer=\"he_normal\", kernel_regularizer=l2(5e-5), activation='relu')(x)\n",
    "    # x = BatchNormalization()(x)\n",
    "    # x = Dropout(top_dropout_rate, name='top_dropout_4')(x)\n",
    "    outputs = Dense(16, activation='softmax', name='pred', kernel_regularizer=l2(5e-5))(x)\n",
    "\n",
    "    model1 = tf.keras.Model(inputs, outputs, name='RestNet-VGG-Inception-ResNet-FineTune')\n",
    "    return model1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c6574006",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metal device set to: Apple M1 Max\n",
      "Model: \"RestNet-VGG-Inception-ResNet-FineTune\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 5, 2500)]         0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 12500)             0         \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 12500)            50000     \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " top_dropout_2 (Dropout)     (None, 12500)             0         \n",
      "                                                                 \n",
      " top_dense_2 (Dense)         (None, 512)               6400512   \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 512)              2048      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " top_dropout_3 (Dropout)     (None, 512)               0         \n",
      "                                                                 \n",
      " pred (Dense)                (None, 16)                8208      \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 6,460,768\n",
      "Trainable params: 6,434,744\n",
      "Non-trainable params: 26,024\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-09 16:00:05.238639: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2022-10-09 16:00:05.238754: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    }
   ],
   "source": [
    "model1 = ftmodel()\n",
    "model1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "96524578",
   "metadata": {},
   "outputs": [],
   "source": [
    "model1.compile(loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'], optimizer = Adam(learning_rate = 0.0003))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "b1f883d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x2956aab20>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.load_weights('./Models/ResNet-VGG-Inception-ResNet-4P/auto-ResNet-VGG-Inception-ResNet-FT-model-weight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "657fc8e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<KerasTensor: shape=(None, 512) dtype=float32 (created by layer 'batch_normalization_1')>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model1.get_layer(index=len(model1.layers)-3).output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "bb009389",
   "metadata": {},
   "outputs": [],
   "source": [
    "model2 = tf.keras.Model(model1.input, model1.get_layer(index=len(model1.layers)-3).output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "8d32cd18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 5, 2500)]         0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 12500)             0         \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 12500)            50000     \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " top_dropout_2 (Dropout)     (None, 12500)             0         \n",
      "                                                                 \n",
      " top_dense_2 (Dense)         (None, 512)               6400512   \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 512)              2048      \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 6,452,560\n",
      "Trainable params: 6,426,536\n",
      "Non-trainable params: 26,024\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "6ee04a13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 82/475 [====>.........................] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-09 16:17:32.048148: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "475/475 [==============================] - 1s 2ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(15200, 512)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_data_train = model2.predict(fine_tune_data_train)\n",
    "xgboost_data_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "eaa3238d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25/25 [==============================] - 0s 2ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(800, 512)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgboost_data_test = model2.predict(fine_tune_data_test)\n",
    "xgboost_data_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e56505e2",
   "metadata": {},
   "source": [
    "### Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70b7e206",
   "metadata": {},
   "source": [
    "The fine-tuning will be done on the ResNet+VGG+Inception-ResNet's PCA reduced $1024D+256D+768D$ representation vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "20df8897",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EPOCHS = 1000\n",
    "# checkpoint_filepaths = ['./Models/ResNet-VGG-Inception-ResNet-4P/auto-ResNet-VGG-Inception-ResNet-FT-model-weight']\n",
    "# model_checkpoint_callback = [tf.keras.callbacks.ModelCheckpoint(\n",
    "#     filepath=checkpoint_filepath,\n",
    "#     save_weights_only=True,\n",
    "#     monitor = \"loss\",\n",
    "#     mode='min',\n",
    "#     save_best_only=True) for checkpoint_filepath in checkpoint_filepaths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "03354278",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x2d90a4d90>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# history = model1.fit(fine_tune_data_train, fine_tune_labels_train, epochs=EPOCHS, batch_size=800, validation_data=(fine_tune_data_test, fine_tune_labels_test), validation_batch_size=800, callbacks=[model_checkpoint_callback])\n",
    "# model1.save_weights(\"./Models/ResNet-VGG-Inception-ResNet-4P/resnet-vgg-inception-resnet-4p-model-weights\")\n",
    "# model1.load_weights('./Models/ResNet-VGG-Inception-ResNet-4P/resnet-vgg-inception-resnet-4p-model-weights')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "86708029",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"./Models/ResNet-VGG-Inception-ResNet-4P-XGBoost/resnet_vgg_inception_xgboost_final.pickle.dat\"\n",
    "def xgb_model(X_train, y_train, X_test, y_test, path):\n",
    "    dtrain = xgb.DMatrix(X_train,label=y_train)\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    \n",
    "    results = {}\n",
    "    \n",
    "    params = {\n",
    "        'max_depth':10,\n",
    "        'eta':0.005,\n",
    "        'objective':'multi:softprob',\n",
    "        'num_class':16,\n",
    "        'eval_metric':'merror'\n",
    "    }\n",
    "    \n",
    "    watchlist = [(dtrain, 'train'),(dtest, 'eval')]\n",
    "    n_round = 400\n",
    "    \n",
    "    model = xgb.train(params, dtrain, n_round, evals=watchlist, evals_result=results, early_stopping_rounds=20)\n",
    "    \n",
    "    pickle.dump(model, open(path, \"wb\"))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "f56fdb21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-merror:0.02901\teval-merror:0.41875\n",
      "[1]\ttrain-merror:0.01954\teval-merror:0.39875\n",
      "[2]\ttrain-merror:0.01770\teval-merror:0.39125\n",
      "[3]\ttrain-merror:0.01592\teval-merror:0.39000\n",
      "[4]\ttrain-merror:0.01441\teval-merror:0.38750\n",
      "[5]\ttrain-merror:0.01362\teval-merror:0.38500\n",
      "[6]\ttrain-merror:0.01257\teval-merror:0.38750\n",
      "[7]\ttrain-merror:0.01178\teval-merror:0.38000\n",
      "[8]\ttrain-merror:0.01007\teval-merror:0.37500\n",
      "[9]\ttrain-merror:0.00875\teval-merror:0.36375\n",
      "[10]\ttrain-merror:0.00836\teval-merror:0.36250\n",
      "[11]\ttrain-merror:0.00816\teval-merror:0.35875\n",
      "[12]\ttrain-merror:0.00704\teval-merror:0.35500\n",
      "[13]\ttrain-merror:0.00625\teval-merror:0.34875\n",
      "[14]\ttrain-merror:0.00572\teval-merror:0.34750\n",
      "[15]\ttrain-merror:0.00539\teval-merror:0.34750\n",
      "[16]\ttrain-merror:0.00513\teval-merror:0.35000\n",
      "[17]\ttrain-merror:0.00507\teval-merror:0.35250\n",
      "[18]\ttrain-merror:0.00480\teval-merror:0.35250\n",
      "[19]\ttrain-merror:0.00447\teval-merror:0.35125\n",
      "[20]\ttrain-merror:0.00447\teval-merror:0.35000\n",
      "[21]\ttrain-merror:0.00447\teval-merror:0.35250\n",
      "[22]\ttrain-merror:0.00447\teval-merror:0.35250\n",
      "[23]\ttrain-merror:0.00441\teval-merror:0.35250\n",
      "[24]\ttrain-merror:0.00421\teval-merror:0.34875\n",
      "[25]\ttrain-merror:0.00388\teval-merror:0.34625\n",
      "[26]\ttrain-merror:0.00355\teval-merror:0.35250\n",
      "[27]\ttrain-merror:0.00349\teval-merror:0.35250\n",
      "[28]\ttrain-merror:0.00336\teval-merror:0.34375\n",
      "[29]\ttrain-merror:0.00316\teval-merror:0.34125\n",
      "[30]\ttrain-merror:0.00316\teval-merror:0.33625\n",
      "[31]\ttrain-merror:0.00296\teval-merror:0.33250\n",
      "[32]\ttrain-merror:0.00283\teval-merror:0.33125\n",
      "[33]\ttrain-merror:0.00283\teval-merror:0.33125\n",
      "[34]\ttrain-merror:0.00289\teval-merror:0.33000\n",
      "[35]\ttrain-merror:0.00289\teval-merror:0.32500\n",
      "[36]\ttrain-merror:0.00289\teval-merror:0.32875\n",
      "[37]\ttrain-merror:0.00263\teval-merror:0.33000\n",
      "[38]\ttrain-merror:0.00243\teval-merror:0.32625\n",
      "[39]\ttrain-merror:0.00230\teval-merror:0.32750\n",
      "[40]\ttrain-merror:0.00224\teval-merror:0.32875\n",
      "[41]\ttrain-merror:0.00224\teval-merror:0.33000\n",
      "[42]\ttrain-merror:0.00217\teval-merror:0.32875\n",
      "[43]\ttrain-merror:0.00224\teval-merror:0.32750\n",
      "[44]\ttrain-merror:0.00237\teval-merror:0.32625\n",
      "[45]\ttrain-merror:0.00224\teval-merror:0.32625\n",
      "[46]\ttrain-merror:0.00224\teval-merror:0.32500\n",
      "[47]\ttrain-merror:0.00217\teval-merror:0.32500\n",
      "[48]\ttrain-merror:0.00217\teval-merror:0.32375\n",
      "[49]\ttrain-merror:0.00211\teval-merror:0.32250\n",
      "[50]\ttrain-merror:0.00211\teval-merror:0.32250\n",
      "[51]\ttrain-merror:0.00211\teval-merror:0.32250\n",
      "[52]\ttrain-merror:0.00204\teval-merror:0.32125\n",
      "[53]\ttrain-merror:0.00197\teval-merror:0.32125\n",
      "[54]\ttrain-merror:0.00191\teval-merror:0.32000\n",
      "[55]\ttrain-merror:0.00191\teval-merror:0.32250\n",
      "[56]\ttrain-merror:0.00184\teval-merror:0.32000\n",
      "[57]\ttrain-merror:0.00171\teval-merror:0.32250\n",
      "[58]\ttrain-merror:0.00164\teval-merror:0.32375\n",
      "[59]\ttrain-merror:0.00158\teval-merror:0.32375\n",
      "[60]\ttrain-merror:0.00145\teval-merror:0.32375\n",
      "[61]\ttrain-merror:0.00145\teval-merror:0.32250\n",
      "[62]\ttrain-merror:0.00145\teval-merror:0.31875\n",
      "[63]\ttrain-merror:0.00138\teval-merror:0.31875\n",
      "[64]\ttrain-merror:0.00138\teval-merror:0.32000\n",
      "[65]\ttrain-merror:0.00132\teval-merror:0.32250\n",
      "[66]\ttrain-merror:0.00132\teval-merror:0.32000\n",
      "[67]\ttrain-merror:0.00132\teval-merror:0.31875\n",
      "[68]\ttrain-merror:0.00132\teval-merror:0.31625\n",
      "[69]\ttrain-merror:0.00125\teval-merror:0.31375\n",
      "[70]\ttrain-merror:0.00125\teval-merror:0.31250\n",
      "[71]\ttrain-merror:0.00118\teval-merror:0.30875\n",
      "[72]\ttrain-merror:0.00118\teval-merror:0.30875\n",
      "[73]\ttrain-merror:0.00118\teval-merror:0.30875\n",
      "[74]\ttrain-merror:0.00118\teval-merror:0.30750\n",
      "[75]\ttrain-merror:0.00118\teval-merror:0.30375\n",
      "[76]\ttrain-merror:0.00125\teval-merror:0.30250\n",
      "[77]\ttrain-merror:0.00125\teval-merror:0.30375\n",
      "[78]\ttrain-merror:0.00125\teval-merror:0.30750\n",
      "[79]\ttrain-merror:0.00125\teval-merror:0.30500\n",
      "[80]\ttrain-merror:0.00118\teval-merror:0.30500\n",
      "[81]\ttrain-merror:0.00118\teval-merror:0.30375\n",
      "[82]\ttrain-merror:0.00118\teval-merror:0.30375\n",
      "[83]\ttrain-merror:0.00118\teval-merror:0.30375\n",
      "[84]\ttrain-merror:0.00118\teval-merror:0.30375\n",
      "[85]\ttrain-merror:0.00118\teval-merror:0.30125\n",
      "[86]\ttrain-merror:0.00112\teval-merror:0.30250\n",
      "[87]\ttrain-merror:0.00112\teval-merror:0.30125\n",
      "[88]\ttrain-merror:0.00112\teval-merror:0.30250\n",
      "[89]\ttrain-merror:0.00112\teval-merror:0.30375\n",
      "[90]\ttrain-merror:0.00112\teval-merror:0.30625\n",
      "[91]\ttrain-merror:0.00105\teval-merror:0.30500\n",
      "[92]\ttrain-merror:0.00105\teval-merror:0.30500\n",
      "[93]\ttrain-merror:0.00105\teval-merror:0.30625\n",
      "[94]\ttrain-merror:0.00099\teval-merror:0.30500\n",
      "[95]\ttrain-merror:0.00086\teval-merror:0.30875\n",
      "[96]\ttrain-merror:0.00086\teval-merror:0.30875\n",
      "[97]\ttrain-merror:0.00079\teval-merror:0.30875\n",
      "[98]\ttrain-merror:0.00079\teval-merror:0.30875\n",
      "[99]\ttrain-merror:0.00079\teval-merror:0.30875\n",
      "[100]\ttrain-merror:0.00079\teval-merror:0.30750\n",
      "[101]\ttrain-merror:0.00079\teval-merror:0.30875\n",
      "[102]\ttrain-merror:0.00079\teval-merror:0.31000\n",
      "[103]\ttrain-merror:0.00072\teval-merror:0.30875\n",
      "[104]\ttrain-merror:0.00072\teval-merror:0.30875\n",
      "[105]\ttrain-merror:0.00072\teval-merror:0.30875\n"
     ]
    }
   ],
   "source": [
    "xgbmodel = xgb_model(xgboost_data_train, fine_tune_labels_train, xgboost_data_test, fine_tune_labels_test, path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03390fa6",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48701107",
   "metadata": {},
   "source": [
    "#### On Train Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0594069",
   "metadata": {},
   "source": [
    "Now, after the model is being trained, we will look at the performance of the model on the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "cf7ded38",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_preds = xgbmodel.predict(xgb.DMatrix(xgboost_data_train, label=fine_tune_labels_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "f2a28269",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pred_labels = np.argmax(train_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "768775b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1, 13, 13, ...,  3,  9,  9])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pred_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "eff52a3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15189"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(train_pred_labels == train_labels['label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fd403fa",
   "metadata": {},
   "source": [
    "Looking at the misclassified images to get a better idea about what features the model is unable to capture."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "8f594a33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3251</th>\n",
       "      <td>3423</td>\n",
       "      <td>./Data/train/train/3423.tif</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3597</th>\n",
       "      <td>3781</td>\n",
       "      <td>./Data/train/train/3781.tif</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5495</th>\n",
       "      <td>5784</td>\n",
       "      <td>./Data/train/train/5784.tif</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5647</th>\n",
       "      <td>5943</td>\n",
       "      <td>./Data/train/train/5943.tif</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8464</th>\n",
       "      <td>8915</td>\n",
       "      <td>./Data/train/train/8915.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11937</th>\n",
       "      <td>12562</td>\n",
       "      <td>./Data/train/train/12562.tif</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12255</th>\n",
       "      <td>12896</td>\n",
       "      <td>./Data/train/train/12896.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12561</th>\n",
       "      <td>13221</td>\n",
       "      <td>./Data/train/train/13221.tif</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12793</th>\n",
       "      <td>13468</td>\n",
       "      <td>./Data/train/train/13468.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14026</th>\n",
       "      <td>14763</td>\n",
       "      <td>./Data/train/train/14763.tif</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14569</th>\n",
       "      <td>15333</td>\n",
       "      <td>./Data/train/train/15333.tif</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id                        images  label\n",
       "3251    3423   ./Data/train/train/3423.tif      4\n",
       "3597    3781   ./Data/train/train/3781.tif      0\n",
       "5495    5784   ./Data/train/train/5784.tif      7\n",
       "5647    5943   ./Data/train/train/5943.tif      4\n",
       "8464    8915   ./Data/train/train/8915.tif      9\n",
       "11937  12562  ./Data/train/train/12562.tif      4\n",
       "12255  12896  ./Data/train/train/12896.tif      9\n",
       "12561  13221  ./Data/train/train/13221.tif      4\n",
       "12793  13468  ./Data/train/train/13468.tif      1\n",
       "14026  14763  ./Data/train/train/14763.tif      4\n",
       "14569  15333  ./Data/train/train/15333.tif      7"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_train_data = train_labels[train_pred_labels != train_labels['label']]\n",
    "mis_train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "2ea8f949",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fscore</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998978</td>\n",
       "      <td>0.999488</td>\n",
       "      <td>978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998919</td>\n",
       "      <td>0.999459</td>\n",
       "      <td>925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.998920</td>\n",
       "      <td>0.994624</td>\n",
       "      <td>0.996767</td>\n",
       "      <td>930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.997925</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998962</td>\n",
       "      <td>962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.998999</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999499</td>\n",
       "      <td>998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997809</td>\n",
       "      <td>0.998904</td>\n",
       "      <td>913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.999006</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999503</td>\n",
       "      <td>1005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997901</td>\n",
       "      <td>0.998950</td>\n",
       "      <td>953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.996736</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998365</td>\n",
       "      <td>916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.996939</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998467</td>\n",
       "      <td>977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    precision    recall    fscore  support\n",
       "0    1.000000  0.998978  0.999488      978\n",
       "1    1.000000  0.998919  0.999459      925\n",
       "2    1.000000  1.000000  1.000000      925\n",
       "3    1.000000  1.000000  1.000000      947\n",
       "4    0.998920  0.994624  0.996767      930\n",
       "5    0.997925  1.000000  0.998962      962\n",
       "6    0.998999  1.000000  0.999499      998\n",
       "7    1.000000  0.997809  0.998904      913\n",
       "8    0.999006  1.000000  0.999503     1005\n",
       "9    1.000000  0.997901  0.998950      953\n",
       "10   0.996736  1.000000  0.998365      916\n",
       "11   1.000000  1.000000  1.000000      955\n",
       "12   0.996939  1.000000  0.998467      977\n",
       "13   1.000000  1.000000  1.000000      943\n",
       "14   1.000000  1.000000  1.000000      928\n",
       "15   1.000000  1.000000  1.000000      945"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = score(train_labels['label'], train_pred_labels)\n",
    "pd.DataFrame({\n",
    "    'precision': precision,\n",
    "    'recall': recall,\n",
    "    'fscore': fscore,\n",
    "    'support': support\n",
    "})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "659ae313",
   "metadata": {},
   "source": [
    "#### On Train-Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "0d1e8f85",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = xgbmodel.predict(xgb.DMatrix(xgboost_data_test, label=fine_tune_labels_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "49233509",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pred_labels = np.argmax(test_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "8f5cba39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11,  8,  0,  7,  7,  6, 13,  8, 14, 15,  6, 15, 13, 14,  1, 10,  0,\n",
       "        9, 11,  1,  7, 14,  3, 12,  6,  0,  3,  5,  3,  3, 15,  1,  6,  2,\n",
       "        6,  8,  4, 15, 11,  9,  6, 12, 14,  2,  5, 12,  5,  1, 13,  6,  0,\n",
       "       10,  8, 15, 11, 14,  9,  9,  9,  2, 15, 13, 10, 11,  6, 10,  8,  0,\n",
       "        6,  3,  8,  4, 12, 12,  8,  6,  7,  0, 13, 11, 11,  0,  4, 12,  9,\n",
       "       12,  1, 13, 13, 13, 10, 11,  4, 10,  7, 15, 11,  8, 13, 10,  3, 10,\n",
       "       12,  9,  7, 14,  8,  2,  5, 15,  7, 10,  9,  4, 15, 12, 14, 13, 11,\n",
       "        6, 13,  1,  0, 12,  1,  5, 11,  5, 15, 13,  3, 11,  6,  1,  6,  9,\n",
       "       14, 10,  7, 14, 12,  8, 13, 12,  8,  6,  8, 13,  2, 13, 15, 14,  4,\n",
       "       14,  4,  9,  4, 14,  7,  7,  8,  4,  5,  3, 13,  1,  2,  3, 12,  8,\n",
       "       13,  6,  3, 15,  7, 12, 10, 14, 14,  8,  6, 10, 13, 11,  6, 13,  4,\n",
       "       14, 11,  7, 15,  0,  3,  0, 10, 11, 13, 10, 11, 11, 11, 13,  9,  1,\n",
       "        9, 11, 13, 14, 12,  1, 12, 13,  7,  6, 13, 10,  4, 12,  8,  0,  9,\n",
       "        1, 15,  1,  7, 10,  2,  2,  9, 10,  7,  0,  3, 12,  3, 15,  2, 14,\n",
       "       10, 12, 14,  0, 12,  8, 13, 10, 15, 10, 11,  1, 10, 13,  1, 15,  8,\n",
       "        1, 10, 15,  5,  0,  9, 14,  6, 13, 12,  9,  1, 14, 11,  1, 14,  3,\n",
       "        0, 11,  4,  3,  1,  4,  6,  2,  7,  8, 10, 10,  8,  5,  7,  8, 13,\n",
       "        7,  8, 15,  0,  4,  3, 12, 10,  0,  8, 13,  5,  7, 15, 10, 12,  5,\n",
       "        3, 13, 10, 10,  9,  2,  5,  3,  0,  6, 14,  9, 14,  7, 13,  2, 15,\n",
       "        9, 14, 15, 10,  0, 15,  3, 11, 13, 15, 15,  1, 12, 11, 13,  0,  4,\n",
       "       13,  0,  3,  3,  2,  8,  2, 15,  1,  3,  3,  2, 12, 14,  8,  0,  3,\n",
       "        8, 12,  1,  3, 13,  5,  2,  3,  7, 15,  0,  3,  2,  6, 15,  3,  6,\n",
       "       15, 14, 14, 15, 12,  8,  2,  6, 12, 13,  6,  9,  3,  7, 10,  1,  2,\n",
       "       11, 14, 13, 14,  8, 11,  0,  1,  2, 10,  1,  0, 14,  6,  9,  0,  2,\n",
       "        5,  3, 14,  5, 13, 11,  4, 14,  7, 10, 12,  7, 13, 14,  5, 11,  8,\n",
       "        0,  1, 11,  0, 11,  3,  7,  2,  5,  6, 15,  9, 13, 13,  0,  0, 11,\n",
       "       15,  0,  3, 13,  5,  5,  6, 14,  7,  8,  4,  8, 13, 12,  1, 15, 11,\n",
       "        9,  8, 10, 14, 12,  1,  5,  1,  9,  9, 15, 14,  7, 14,  9,  5, 11,\n",
       "       12, 10,  4,  8,  9,  3,  7,  8,  6, 10,  1, 10,  7, 10,  8, 11, 10,\n",
       "        7,  3,  1, 10,  0,  7,  8,  5,  9, 14,  6,  5, 15,  0,  1,  7, 11,\n",
       "       12,  1,  0,  6,  7,  1, 10,  7,  9,  7, 14,  0,  5, 14,  6, 14,  7,\n",
       "        1, 14, 14, 14,  6,  5, 12, 14,  1,  6,  2,  3,  7,  2,  0,  8,  1,\n",
       "        0,  8,  6,  5, 14,  3,  6, 11,  8, 14,  6,  9,  7,  9, 11, 14, 13,\n",
       "        3, 10,  9, 10, 15,  2,  7,  5, 14,  9,  1,  1,  9, 13, 10, 10, 12,\n",
       "        9, 11,  6, 11,  5,  8, 11, 14, 14,  9, 15, 15,  8,  5,  8,  6,  5,\n",
       "       14,  5, 15, 15, 14,  0,  3,  2, 12,  5, 15, 11, 10,  5, 13,  1, 11,\n",
       "        1,  7,  3,  4,  4, 12, 11, 15, 14, 13,  4,  8,  5, 12, 13,  4,  0,\n",
       "       13,  0,  1, 14, 13,  7,  6, 12,  8,  7,  5, 15,  5,  9,  4,  1, 14,\n",
       "        1, 13,  6,  6,  3,  7,  7,  5,  5, 12,  6,  8,  0,  4,  7,  7,  0,\n",
       "       12,  6,  7,  0,  3, 11,  2,  3, 12,  4,  6,  1,  0, 10, 14, 15, 10,\n",
       "       11, 12, 10,  7,  2, 12, 15,  0, 11,  6,  0,  2,  5, 13,  7,  6,  3,\n",
       "        1,  3,  2, 14,  2,  8,  6, 12,  8, 13, 13, 14,  4,  5,  2,  5, 14,\n",
       "       12, 13, 13,  4, 12,  0,  1, 15,  8, 10, 13,  8,  4,  9,  2,  4, 15,\n",
       "        1,  2,  2, 15,  0,  3,  9,  8, 14,  8,  6,  7, 11, 11, 13,  7,  3,\n",
       "        7,  7, 13,  8, 12,  1, 12, 15,  7, 10,  1,  6,  5,  6, 14, 13,  2,\n",
       "       10,  3, 12,  6,  5,  7,  2,  8, 10, 13,  2,  2,  8, 14, 14, 14, 12,\n",
       "       11,  1, 10, 15,  1, 11, 15,  0,  9,  9,  5, 13, 10,  2,  2,  9, 15,\n",
       "        6])"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "ee078822",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "553"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(test_pred_labels == test_labels['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "c3db9b5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>images</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33</td>\n",
       "      <td>./Data/train/train/33.tif</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>41</td>\n",
       "      <td>./Data/train/train/41.tif</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65</td>\n",
       "      <td>./Data/train/train/65.tif</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>67</td>\n",
       "      <td>./Data/train/train/67.tif</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>100</td>\n",
       "      <td>./Data/train/train/100.tif</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783</th>\n",
       "      <td>15589</td>\n",
       "      <td>./Data/train/train/15589.tif</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>784</th>\n",
       "      <td>15596</td>\n",
       "      <td>./Data/train/train/15596.tif</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>786</th>\n",
       "      <td>15660</td>\n",
       "      <td>./Data/train/train/15660.tif</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788</th>\n",
       "      <td>15687</td>\n",
       "      <td>./Data/train/train/15687.tif</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>15832</td>\n",
       "      <td>./Data/train/train/15832.tif</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>247 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                        images  label\n",
       "2       33     ./Data/train/train/33.tif      6\n",
       "3       41     ./Data/train/train/41.tif      1\n",
       "4       65     ./Data/train/train/65.tif     14\n",
       "5       67     ./Data/train/train/67.tif      9\n",
       "7      100    ./Data/train/train/100.tif     10\n",
       "..     ...                           ...    ...\n",
       "783  15589  ./Data/train/train/15589.tif      5\n",
       "784  15596  ./Data/train/train/15596.tif      8\n",
       "786  15660  ./Data/train/train/15660.tif      4\n",
       "788  15687  ./Data/train/train/15687.tif     12\n",
       "797  15832  ./Data/train/train/15832.tif     11\n",
       "\n",
       "[247 rows x 3 columns]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_train_data = test_labels[test_pred_labels != test_labels['label']]\n",
    "mis_train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "b538b721",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>fscore</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.771930</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>0.792793</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.653061</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.688172</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.886364</td>\n",
       "      <td>0.906977</td>\n",
       "      <td>0.896552</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.913043</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.885714</td>\n",
       "      <td>0.861111</td>\n",
       "      <td>0.873239</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.717391</td>\n",
       "      <td>0.660000</td>\n",
       "      <td>0.687500</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.928571</td>\n",
       "      <td>0.847826</td>\n",
       "      <td>0.886364</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.854167</td>\n",
       "      <td>0.788462</td>\n",
       "      <td>0.820000</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.844828</td>\n",
       "      <td>0.907407</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.812500</td>\n",
       "      <td>0.829787</td>\n",
       "      <td>0.821053</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.844444</td>\n",
       "      <td>0.655172</td>\n",
       "      <td>0.737864</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.840000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.792453</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.649123</td>\n",
       "      <td>0.860465</td>\n",
       "      <td>0.740000</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.767123</td>\n",
       "      <td>0.875000</td>\n",
       "      <td>0.817518</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.964912</td>\n",
       "      <td>0.859375</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.723404</td>\n",
       "      <td>0.790698</td>\n",
       "      <td>0.755556</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    precision    recall    fscore  support\n",
       "0    0.771930  0.814815  0.792793       54\n",
       "1    0.653061  0.727273  0.688172       44\n",
       "2    0.886364  0.906977  0.896552       43\n",
       "3    0.954545  0.913043  0.933333       46\n",
       "4    0.885714  0.861111  0.873239       36\n",
       "5    0.717391  0.660000  0.687500       50\n",
       "6    0.928571  0.847826  0.886364       46\n",
       "7    0.854167  0.788462  0.820000       52\n",
       "8    0.844828  0.907407  0.875000       54\n",
       "9    0.812500  0.829787  0.821053       47\n",
       "10   0.844444  0.655172  0.737864       58\n",
       "11   0.840000  0.750000  0.792453       56\n",
       "12   0.649123  0.860465  0.740000       43\n",
       "13   0.767123  0.875000  0.817518       64\n",
       "14   0.964912  0.859375  0.909091       64\n",
       "15   0.723404  0.790698  0.755556       43"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = score(test_labels['label'], test_pred_labels)\n",
    "pd.DataFrame({\n",
    "    'precision': precision,\n",
    "    'recall': recall,\n",
    "    'fscore': fscore,\n",
    "    'support': support\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ed4db76c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8141554511573338"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(fscore)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "067c5096",
   "metadata": {},
   "source": [
    "#### On Validation Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e665591",
   "metadata": {},
   "source": [
    "Generating Predictions for the Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "4b085c0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>images</th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>850</th>\n",
       "      <td>./Data/validation/validation/17801.tif</td>\n",
       "      <td>17801</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>./Data/validation/validation/17802.tif</td>\n",
       "      <td>17802</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>852</th>\n",
       "      <td>./Data/validation/validation/17803.tif</td>\n",
       "      <td>17803</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>./Data/validation/validation/17804.tif</td>\n",
       "      <td>17804</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>750</th>\n",
       "      <td>./Data/validation/validation/17805.tif</td>\n",
       "      <td>17805</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>./Data/validation/validation/18696.tif</td>\n",
       "      <td>18696</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>./Data/validation/validation/18697.tif</td>\n",
       "      <td>18697</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>760</th>\n",
       "      <td>./Data/validation/validation/18698.tif</td>\n",
       "      <td>18698</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>787</th>\n",
       "      <td>./Data/validation/validation/18699.tif</td>\n",
       "      <td>18699</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>./Data/validation/validation/18700.tif</td>\n",
       "      <td>18700</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     images     id  label\n",
       "850  ./Data/validation/validation/17801.tif  17801     -1\n",
       "898  ./Data/validation/validation/17802.tif  17802     -1\n",
       "852  ./Data/validation/validation/17803.tif  17803     -1\n",
       "798  ./Data/validation/validation/17804.tif  17804     -1\n",
       "750  ./Data/validation/validation/17805.tif  17805     -1\n",
       "..                                      ...    ...    ...\n",
       "246  ./Data/validation/validation/18696.tif  18696     -1\n",
       "211  ./Data/validation/validation/18697.tif  18697     -1\n",
       "760  ./Data/validation/validation/18698.tif  18698     -1\n",
       "787  ./Data/validation/validation/18699.tif  18699     -1\n",
       "891  ./Data/validation/validation/18700.tif  18700     -1\n",
       "\n",
       "[900 rows x 3 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_data = pd.DataFrame({'images':['./Data/validation/validation/'+name for name in os.listdir('./Data/validation/validation/')]})\n",
    "validation_data['id'] = [name.split('.')[0] for name in os.listdir('./Data/validation/validation/')]\n",
    "validation_data['label'] = -1 # Simply added to prevent re-writing code\n",
    "validation_data.sort_values(by=['id'], inplace=True)\n",
    "validation_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "53635253",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 2048)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_resnet = np.load('./Data/ResNet-4P-validation-precompute/whole_validation_data_precomp.npy')\n",
    "fine_tune_data_validation_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "8e78b6f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 512)"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_vgg = np.load('./Data/VGG16-4P-validation-precompute/whole_validation_data_precomp.npy')\n",
    "fine_tune_data_validation_vgg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "0823f8b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 1536)"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_inception_resnet = np.load('./Data/Inception-ResNet-4P-validation-precompute/whole_validation_data_precomp.npy')\n",
    "fine_tune_data_validation_inception_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "f6dce158",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 1300)"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_red_resnet = np.zeros((fine_tune_data_validation_resnet.shape[0], 5, N_COMPS_1))\n",
    "for i in range(5):\n",
    "    fine_tune_data_validation_red_resnet[:,i,:] = pcas_resnet[i].transform(fine_tune_data_validation_resnet[:,i,:])\n",
    "fine_tune_data_validation_resnet = fine_tune_data_validation_red_resnet\n",
    "del fine_tune_data_validation_red_resnet\n",
    "fine_tune_data_validation_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "62c9d2ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 300)"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_red_vgg = np.zeros((fine_tune_data_validation_vgg.shape[0], 5, N_COMPS_2))\n",
    "for i in range(5):\n",
    "    fine_tune_data_validation_red_vgg[:,i,:] = pcas_vgg[i].transform(fine_tune_data_validation_vgg[:,i,:])\n",
    "fine_tune_data_validation_vgg = fine_tune_data_validation_red_vgg\n",
    "del fine_tune_data_validation_red_vgg\n",
    "fine_tune_data_validation_vgg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "980446d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 900)"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation_red_inception_resnet = np.zeros((fine_tune_data_validation_inception_resnet.shape[0], 5, N_COMPS_3))\n",
    "for i in range(5):\n",
    "    fine_tune_data_validation_red_inception_resnet[:,i,:] = pcas_inception_resnet[i].transform(fine_tune_data_validation_inception_resnet[:,i,:])\n",
    "fine_tune_data_validation_inception_resnet = fine_tune_data_validation_red_inception_resnet\n",
    "del fine_tune_data_validation_red_inception_resnet\n",
    "fine_tune_data_validation_inception_resnet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "586ac8df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 5, 2500)"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fine_tune_data_validation = np.concatenate([fine_tune_data_validation_resnet, fine_tune_data_validation_vgg, fine_tune_data_validation_inception_resnet], axis=-1)\n",
    "fine_tune_data_validation.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "db7d1256",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29/29 [==============================] - 0s 4ms/step\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-09 14:57:12.599361: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    }
   ],
   "source": [
    "valid_preds = model1.predict(fine_tune_data_validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "18b17d2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_pred_labels = np.argmax(valid_preds, axis=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bece594",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "304c47de",
   "metadata": {},
   "source": [
    "For Kaggle Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "e7ca9d19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17801</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17802</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17803</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17804</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17805</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895</th>\n",
       "      <td>18696</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>18697</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>18698</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>18699</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>18700</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  label\n",
       "0    17801      4\n",
       "1    17802      6\n",
       "2    17803      4\n",
       "3    17804      8\n",
       "4    17805      3\n",
       "..     ...    ...\n",
       "895  18696      7\n",
       "896  18697     12\n",
       "897  18698      8\n",
       "898  18699      4\n",
       "899  18700      4\n",
       "\n",
       "[900 rows x 2 columns]"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_data = pd.DataFrame({'id':list(range(17801, 18701))})\n",
    "submission_data['label'] = valid_pred_labels\n",
    "submission_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "6eea7d9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_data.to_csv('./Submissions/submission-resnet-vgg-inception-4p.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}