2-product-detection/[v1] Product Detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras as tfk\n",
    "\n",
    "from tensorflow.keras.preprocessing import image as tfk_image\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator as tfk_ImageDataGenerator\n",
    "\n",
    "import datetime, os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns; sns.set();\n",
    "from PIL import Image\n",
    "\n",
    "from sklearn.utils import class_weight\n",
    "from sklearn.metrics import confusion_matrix, log_loss, accuracy_score\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "EPS = 1e-9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.python.keras.applications.efficientnet import EfficientNetB7 as tfk_BaseModel\n",
    "from tensorflow.python.keras.applications.efficientnet import preprocess_input as tfk_preprocess_input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_train = pd.read_csv('shopee-product-detection-dataset/train.csv').sort_values(by=['filename'])\n",
    "pd_test  = pd.read_csv('shopee-product-detection-dataset/test.csv').sort_values(by=['filename'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = (256, 256, 3)\n",
    "# input_shape = (512, 512, 3)\n",
    "batch_size=128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 94869 images belonging to 42 classes.\n",
      "Found 10523 images belonging to 42 classes.\n"
     ]
    }
   ],
   "source": [
    "train_datagen = tfk_ImageDataGenerator(\n",
    "    preprocessing_function=tfk_preprocess_input,\n",
    "    validation_split=0.1,\n",
    "    #featurewise_center=True,\n",
    "    #featurewise_std_normalization=True,\n",
    "    rotation_range=45,\n",
    "    shear_range=20,\n",
    "    zoom_range=[0.9, 1.1],\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    horizontal_flip=True,\n",
    "    vertical_flip=True\n",
    ")\n",
    "\n",
    "train_generator = train_datagen.flow_from_directory(\n",
    "    'shopee-product-detection-dataset/train/',  # this is where you specify the path to the train data folder\n",
    "    target_size=input_shape[:2],\n",
    "    color_mode='rgb',\n",
    "    batch_size=batch_size,\n",
    "    class_mode='categorical',\n",
    "    shuffle=True,\n",
    "    subset='training'\n",
    ")\n",
    "\n",
    "val_generator = train_datagen.flow_from_directory(\n",
    "    'shopee-product-detection-dataset/train/',  # this is where you specify the path to the train data folder\n",
    "    target_size=input_shape[:2],\n",
    "    color_mode='rgb',\n",
    "    batch_size=batch_size,\n",
    "    class_mode='categorical',\n",
    "    shuffle=True,\n",
    "    subset='validation'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 12186 validated image filenames.\n"
     ]
    }
   ],
   "source": [
    "test_datagen = tfk_ImageDataGenerator(\n",
    "    preprocessing_function=tfk_preprocess_input\n",
    ")\n",
    "\n",
    "test_generator = test_datagen.flow_from_dataframe(\n",
    "    pd_test,\n",
    "    directory='shopee-product-detection-dataset/test/',\n",
    "    x_col='filename',\n",
    "    y_col='category',\n",
    "    target_size=input_shape[:2],\n",
    "    color_mode='rgb',\n",
    "    batch_size=batch_size,\n",
    "    class_mode='raw',\n",
    "    shuffle=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x,y = next(iter(test_generator))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "813"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model = tfk_BaseModel(input_shape=input_shape, weights='imagenet', include_top=False)\n",
    "len(base_model.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for layer in base_model.layers[:-16]:\n",
    "  layer.trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tfk.Sequential([\n",
    "    base_model,\n",
    "    tfk.layers.GlobalMaxPooling2D(),\n",
    "    tfk.layers.Dense(512, activation='relu'),\n",
    "    #tfk.layers.Dense(256, activation='relu'),\n",
    "    tfk.layers.Dense(128, activation='relu'),\n",
    "    tfk.layers.Dense(42, activation='softmax', name='output_layer'),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "efficientnetb7 (Model)       (None, 8, 8, 2560)        64097687  \n",
      "_________________________________________________________________\n",
      "global_max_pooling2d (Global (None, 2560)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 512)               1311232   \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               65664     \n",
      "_________________________________________________________________\n",
      "output_layer (Dense)         (None, 42)                5418      \n",
      "=================================================================\n",
      "Total params: 65,480,001\n",
      "Trainable params: 6,759,754\n",
      "Non-trainable params: 58,720,247\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAIjCAYAAAD4N1uCAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3df2wb533H8c9Zktt1yOykgLwmgY0ahoImQ5V/mrUJ5tSGszXNTjEa/5AoxyoCO6AQD8g6o2gHCu6QbUUBGkuHAjaoFPnDkCjYGDpI2PrPJMBGUXlBAzAb0FVG2pXOUoxEgZEYViChjWd/pM/5SB0pkiJ51KP3CyBsHu+e+97x+OHxeY6UZ4wxAgC45NyOuCsAAHQe4Q4ADiLcAcBBhDsAOGgw7gJ6YWlpSVeuXIm7DAAxGxgY0N/93d/p93//9+Mupeu2xZn7wsKCrl27FncZAGK2sLCglZWVuMvoiW1x5i5JiURCc3NzcZcBIEae58VdQs9sizN3ANhuCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe6bdPPmTU1PT8vzPE1PT+udd96JnD49Pa2ZmZmW2p6ZmWl5mX5RLBa1sLCgsbGxnq53K+8zoJMI901YWVnRF77wBX3zm9+UMUZPP/20ZmZmIqe///77cZcrSSqXy135Tevadi9cuKCJiQktLS01tVzUbWFhoeN1dls7+7fe9sehtv5+qg0tMttAIpEwiUSi4+0mk0kTtQvrTe8Hi4uLXaktql1JG65rdXU1mK/2VigUOl5nt7W7f0ulUrDdpVKpC5U1J6r+QqHQF7V1giQzNzcXdxm98Apn7ptw+fLllqbHrVwua3Z2tq/a/eUvf6l8Pi9jTHArFApKpVIaHh7ucKXdtZn9sGvXrsj/91K9+sPPQ1y1oXWEewPFYlEXL16U53kaGxsL/vZi7UfTeh9Z7f16/c/lclkLCwvBfOEXVr1l6tVUO//S0lIwz+3btyVJ6XQ66CapV1vUchutO6rdestNT09XtXv48GHt3bu3av6VlRUdO3aszrNSX+22NLNtxWJRS0tLwTyzs7NBnbdu3QrabvT82mn19kO74wD9Un8r7BuEXX5mZqbq+be3ixcvBsuEHwtvV73j3G5vuVxuayxr24j5o0NPtNMtUygUjO/7JpvNGmOMWV5eNpJMLpcL5lGdbofa6b7vR87r+75JpVLB/WQyGdyPWqZRTeH5V1dXjTHG5PN5I8kkk8mmamu03Eb7I2r7atu1bWiDLpfweltRu8+a2Tb7eHieUqkUdK2tra0Ftdduo20rPC1qP6RSqarnuZ52npte1N9oei273kKhsK5W2wUX9fz6vh8cE60c57lcrqXjRduoW4ZwryObzUaGVfhF2soLoXaabT8ccqurq8b3/Q2XqVdTM+ttZp5urdsYY9bW1owkk8lkTJRcLhe8qNvRzvZGzZPL5Ywkk06nN91Wu7X3U/3NblcqlWp4MpFOp40kk8/nq2oNP+fNHmvt9P8T7o5pJ9zDZwi1N2sz4W7bb6TRmVxUTd0M906se6PpxnwUDpsZSO1UuHe6rXZq76f6W92ufD4fBHl4OfumE35zT6fTVWHfzrHWrO0U7vS512H7Hk1ooM/eOtl+P9UU97qLxaIkbbmBVFSbnZ3VuXPn5Pv+usdGR0eVTCb18ssvq1wuq1wu6913360ad4nzOHcJ4b6B8KBUJ9kD337pqRXdqqmX604mk+umtTuQ2k1RdW4lvap/enpakrSwsKCXX35Z3/ve9zQyMtKwph/+8Ie6ceOGpqamIueL8zh3AeFeRyaTkSRduXJF5XJZ0r0R/E6w4X758uWg/du3bwcvkjhqaqRT67ZvZk8//fS6x65fv67R0dFNVtoZNli+/OUvx1xJe3pZ/82bN4Pnc2JiQpLWXQEVZs/eJyYmNDs7q89//vNVj8d5nDult91A8Wj3ahlF9PnZvkHbdyjduyKh3vRwW7VXBITbTiaTGy5Tr6aoL5qEvxhj2whfrZJOp5tebqP9UdtueNry8nLVNocH+cL7bTMDqVH7rNlts/ft+kulkkmlUlWD28aYdVeghL+AZQcRo/ZDM1fLRH2JqV/qj7rSxrJt2Kum7PL5fD4YPA/XWrtc1MB6s8d5O7SN+twJ9wby+bxJpVLBwW+DLOrAa+VmFQqFoP1UKlX1JlFvmWZrqjfNvvnYgctml2u07qh2reXl5eAFn0wmg6CvtdmB1Ki6m902+//wpXaZTGbd1Rj5fD54fHFx0Rhjgkv2bO1R+2GjcG/luOl1/c3WZtdVu7y9eiZ8rFi+71cd87W1bnSc1755NWM7hbtnjPujFJOTk5Kkubm5mCtBP7Jf1tmqL4WtWH+5XNY3vvENXbp0qafr9TxPc3NzSiQSPV1vDM7R5w6g565evarjx4/HXYbTCHdsa/byy9r/bxVbqf6ZmZmqnxk4fPhw3CU5bTDuAoCNNPsbJ+10S+zZs6fq/1upa0PaWvXbK2gymYzOnj0bczXuI9zR97oZWP0chs3YSvWfPXuWUO8humUAwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQActG1+FXJ+fl6VSiXuMgCgJ7ZFuI+PjxPs21yxWNTPfvYzHTx4MO5SEKPx8fFt80dCtsXfUAXm5+c1OTm5pX7/HNgE/oYqALiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4aDDuAoBuOHPmjH7yk59o9+7dkqRf//rXGhwc1Be/+MVgnl/96lf67ne/q2effTamKoHuIdzhpO9///uR069fv151/+bNm4Q7nES3DJz0rW99S0NDQxvOd/LkyR5UA/Qe4Q4njY+Pq1KpNJznscce06OPPtqjioDeItzhpEceeUSf/exn5Xle5ONDQ0M6depUj6sCeodwh7OmpqY0MDAQ+didO3c0MTHR44qA3iHc4ayTJ0/q7t2766bv2LFDTzzxhPbt2xdDVUBvEO5w1kMPPaQnn3xSO3ZUH+ae52lqaiqmqoDeINzhtNOnT0f2u7/wwgsxVAP0DuEOpx07dqwq3AcGBnTo0CENDw/HWBXQfYQ7nPbAAw/omWeeCQZWjTE6ffp0zFUB3Ue4w3mnTp2SMUbSR5dAHj16NOaKgO4j3OG8559/Xjt37pQkPffcc7rvvvtirgjoPn5bpg+89957unnzZtxlOG3//v366U9/qv379+vatWtxl+OsgYEBjY2NaXCQaImbZ+znVcTmpZde0ptvvhl3GUBH/OAHP6DrK37neHvtAx988IESiYTm5ubiLgXYFM/z9Jvf/CbuMiD63AHASYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3x928eVPT09PyPE/T09N65513IqdPT09rZmampbZnZmZaXqZfFItFLSwsaGxsLO5SeqLV7d1u+8dFhLvDVlZW9IUvfEHf/OY3ZYzR008/rZmZmcjp77//ftzlSpLK5bI8z+t6uxcuXNDExISWlpaaWi7qtrCw0PT6a5dt9Je3bt68uW7+zWp2e9udH33IIHaJRMIkEomOt5tMJk3UU1xvej9YXFzsSm1R7UracF2rq6vBfLW3QqHQUg35fD5YNplM1p3PPj/trKORZrZ3M/PbZebm5lotDZ33CmfuDrt8+XJL0+NWLpc1OzvbV+3+8pe/VD6flzEmuBUKBaVSKQ0PD7fU1t69eyVJ6XRaly9f1u3bt9fNc/v2bR04cCC43+o6AItw38KKxaIuXrwoz/M0NjamlZUVSVr3Ub7eR3x7v17/arlc1sLCQjBfOCDrLVOvptr5l5aWgnlsyKXT6aAboF5tUctttO6odustNz09XdXu4cOHg1C2VlZWdOzYsapprYw/HDlyRJL04x//eN1jP/7xj4PHo0Q9J8ViseF8Y2NjunXrVmR79fYZHBDzRweY9rplCoWC8X3fZLNZY4wxy8vLRpLJ5XLBPKrzsbp2uu/7kfP6vm9SqVRwP5lMBvejlmlUU3j+1dVVY8y9bopwF0Wj2hott9H+iNq+2nZtG9qgOySqSyWVSlXtq3psDY26zOrVa8xH+yOTyVTV6/u+KZVK6+ZLJpPB9Gw229Lz1aiGjbaPbpm+8Arh3gfaCXf7Yg2TVBUwzYZ71DTbfjjkVldXje/7Gy5Tr6Zm1tvMPN1atzHGrK2tGUlBgNbK5XJBGLbDrs8GqX1jsW0vLy/Xrc0uU/ucSKqqyY4vrK2tBdNKpVJH9lkz20e49wXCvR+0E+7hM9ram7WZcLftN9LoLDuqpm6GeyfWvdF0Yz46Q9/MIGfttoY/BWz0xhx1tm9DO/ymW+9TQSf2WTPbR7j3BQZUtyrbh2xCA3321sn2+6mmuNdt+7Y7NciZzWaDgdVisajHHnus4fxRA+G7du2SVP18NTtgHufzhe4j3Le4egNlm+X7viQFX3pqRbdq6uW6k8nkumlRA6mb8eSTT0r6aBB1ZWUluF+PfU6iBlCj6m1WnM8Xuodw36IymYwk6cqVKyqXy5LuXfnQCTZILl++HLR/+/ZtTU9Px1ZTI51at30ze/rpp9c9dv36dY2Ojm6y0nv27t2rVCqliYkJvf/+++uuyqmVSCQkSb/4xS+CaXZbjx8/Hkyz+2KjN+Y4ny/0QG+7gRCl3atlFNFXms/njTEfDc7ZaeGBtajp4bZsf3L4yhF7SyaTGy5Tr6bwY/YKjvAgn20jfLVKOp1uermN9kdtu+FpdhDTbrN9PGyjgdRmrpaxNYb77O3zEb7KKWrf2u22V8fY6dlsdt3VO/ZqIt/3g+23g7H2eWzl+WpljEH0ufcLBlT7QbvfUM3n8yaVSgUvWPtCjnrBtnKzCoVC0H4qlap6k6i3TLM11Ztmw84OXDa7XKN1R7VrLS8vByGfTCaDoK+10UDqRuHeaD9HXQra6DnJZDLB9Gw2u+4ySLsv7MBqMpmsuuwxvB2tPF/NINz7xiueMYyexG1yclKSNDc3F3MlwOZ4nqe5ubmgCwmxOUefOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADhoMO4C8JFr167p6NGjcZcBwBGEex/49Kc/rUqlohMnTsRdCrBpBw4ciLsESOJvqGJbmJ+f1+TkpDjcsU3wN1QBwEWEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBg3EXAHTD8vKyfv7znwf333rrLUlSJpOpmu9LX/qS9u7d29PagF7wjDEm7iKATvM8T5I0NDQkSTLGyBijHTvufVitVCr6+te/ru985zux1Ah00Tm6ZeCkl156SUNDQ6pUKqpUKrpz547u3r0b3K9UKpKkQ4cOxVwp0B2EO5w0MTERBHg9999/v44cOdKjioDeItzhpEOHDumTn/xk3ceHhoY0Pj6uwUGGneAmwh1OGhgY0KlTp7Rz587IxyuVihKJRI+rAnqHcIezEomEPvzww8jHHnzwQT311FM9rgjoHcIdzvrc5z6nhx9+eN30oaEhnT59OriiBnAR4Q5neZ6nqamp4HJIq1KpaHx8PKaqgN4g3OG0RCKx7qqZAwcOaHR0NKaKgN4g3OG0Rx99VJ/5zGeC+0NDQ/rqV78aX0FAjxDucN7p06eDrpk7d+5oYmIi5oqA7iPc4byJiQnduXNHkvT4449r//79MVcEdB/hDuft27cv6GOfmpqKuRqgN9b9cNhbb72lP/zDP4yrHgBAC3bu3KkPPvigdvK5dd+9fvfddyVJV69e7UFZQG/cvXtXxWJRn/rUp+IuBeiY+fl5/eM//mPkY3V/WOP48eNdKwgAsHmVSqVuuNPnDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAc1JNwLxaLWlhY0NjYWE+W61Y7uCdqn87MzGhmZibGqqr18nnnGO8vW+H47DpTY25uzkRM3pRkMmkktdxuu8t1qx3cE7VPU6mUSaVSHV9XPp8P1pdMJs3y8nLbNbYql8uZVCoVtJNKpczq6qoplUpV7fbbMW7/b2+rq6t1l11dXV03fyfUtmlvvu+bTCZjCoVCR9YTpZ+Oz3r7QZJJp9NmcXHRlEqlttbdIK9f6Um4G2PaPmg6dbAR7p3Xi31aKpXM4uJi8P9sNmskBdM2spkaU6mUSSaTJpfLVdWzuroavJg7sa5uHeP5fD6Ylkwm6y4XDsJOB26hUIisy75hrq2tdXR9Yf10fIb3QzjIc7mc8X3f+L7f1r4n3DvYDu7pxT6NCvFW1ttujel02vi+X/fxXC7X9+Fup6XTaSPJ5PP5dcvk8/ng8W49l1Ft27Br9KbTjfV2WivHZ73phUIhCPhWz+B7Eu7Ly8vG9/3gYKp9F4rasPA7naTIj2rhMwp7ECaTyXUHaqlUMplMpurjc7itdp7oQqFgstls8CJfXFxct35bf21NjeqJ+gjc7sfiQqFgFhcXgxrtOpPJZORZUTP7vNn5wrXW7qt6+873/XXP3UbHTq16oRCu2fd9s7a21tZHcxvcjbp/artlbF39doxLCrYnm82u245sNhs8HlV7J47hVsLO5eOz0Wt7eXk58ox/I10Pd7tjbL9eeKfXPvFhtu/NmPrvXnY527adz74YLPvRslAoBB9Hwzu4nXC367EvEGPu9U8mk8mgpqj1bVSPfdHYbbDbFe4CaEZ4P9t6SqVSsP7agG9mnzc7X3ifhvdV7f1G+6mZYyfMhmrUi8D3fZNMJoMaw21ZzYS7DdhWz6L68Ri396O6kez0est26hiuF+JRIejy8dkog+rtj410PdzrHVTpdLruPPadKnzw2uAMn2FEtW3PyOyTa8y9/tF6y7UT7o22baNpG9VjTPWLp5kzglZqtGdj4eeg2X3e7nPTzD5vdp5w3WH2LKr2xW5fhOE3M/uC6cTxXPtY7S1quX44xu19W0t4YDWXywWfTqKW7dQxbJezoV8qlYI+93A9Lh+f9dpq5fEoXQ/3ZgaXau9HLWNfjOG+znobXG96vT7EXof7RvUYc6/f0XYhtKvZfdTsPm/3uWnnxdPqwKTv+5FXftQ7M+10uBtTPTjWqFukH47x2sfDYR3+BNNomzd7DEe9GaZSqXVn+C4fnxst18zjUboe7rV9elFnjc2GbbvzGfPRx8R6fa1xhHujeiz7Ma/RpWrt1Bg1vdvztfPiaebYsbLZbNWZbKN2N5reiH1BRw1ANmq3H4/x8H17rOXz+aDPeaMaOnEMN/scuHx8NqrbmHtvTq1eqtmTAdXFxcXg3d33/XWDN7UbFtWnaOdrpq+8dr7wgRu1XK/DfaN6jDHBR1m73zrZLWOnh/dRs/u83eemnRePMRsfO8bcu968nk6Gu/3YH1VHo3b78RgP37d9ytls1mSz2ao3r6hlO3UMN/scuHx81mvbssdcs9/hsHoyoLrR4FO98Au/29t3r/AGRu2QqNH/dp7IZrQb7s2s3777l0qlYDCwHVFt2zOt8MBOs/u83eemnRdPM8eODZCwXC4XObjXzGBeM+zZe70B7lbe4OM8xmvv277u2v3ZzjFtTHPHcLPPgcvHZ7312eXtgHCrejagWntLJpOmUChE9lHagyF88X42m40cPQ8/aXZH1O5MO18+n6/6CFlv/c2I+uJBVFtR0xrVYweUwgdNux/LjLm3/20Q2PZrD5Zm93kz89Vuc6P7djvDA5z1LqmLOnbCVzaEb+E3LntWGr6UzZ4N2faMaf5bioVCIQjC5eXldV88qX2h9uMxbqeFj3lbe/hNq97roxPHcNRzXo/Lx2e47S31JSZbYL2dUDvNKhQKVdfRZrPZyHfJ8HWm9b5+bg9aey2uHekPf0uv2TMIK2q5Zqc1W0+jdbVaZ/h5yGQykfuy2X2+0Xz1Dvp6t0b7qdGxE3X82FvtAF7t18DtCy+bzQYvnFa/gp7L5aoGE+1zWntG12/HeL3nwBgT2XXRjWN4ozqiuHh8NlpvOp3e1Hhbo3D3frvywPz8vCYnJ1UzuaFbt27p4x//uPbu3btu+iOPPNJSW2id53mStCX3M8cO+lm/H58N8vrcpn8VcmFhQSMjI+s2XpL27NmjbDa72VXAURw76Gdb/fgc3GwD8/Pz+t///V/9yZ/8SdVOuHXrlq5fv66zZ89udhVooFgsVv1/eHg4xmpaw7GDfrbVj89Nn7lfuXJF9913n7797W/L8zx5nqeZmRn913/9V99uvK1zo9tWqHHPnj3BMuH/bwVb8djB9rHVj8+O9LkDAHqvq33uAID+Q7gDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEHrfs/9E5/4hCTF/pO3AID2rQv3P/3TP9U//MM/6O7du3HUA3TFj370I/393/+9rl69GncpQEc9/PDDkdPXhfvg4KC+8pWvdL0goJcqlYok6fjx4zFXAvQGfe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABw3GXQDQDR9++KH+7//+L7hv//8///M/VfPdf//9Pa0L6BXCHU762Mc+Fjn9gQceqLr/2muvKZVK9aIkoKfoloGTHnvssabmGx4e7nIlQDwIdzjpa1/7mgYGBhrOMzg4qGPHjvWoIqC3CHc46Stf+Yp27Kh/eA8MDOiZZ55Z100DuIJwh5N2796tZ599VoOD0cNKxhidOnWqx1UBvUO4w1kvvvii7t69G/nYzp079fzzz/e4IqB3CHc467nnntPHP/7xddOHhoZ09OhR/e7v/m4MVQG9QbjDWb/zO7+jF154QUNDQ1XTK5WKJicnY6oK6A3CHU6bnJxUpVKpmvZ7v/d7+uM//uOYKgJ6g3CH044cOVL1LdShoSGdPHlSO3fujLEqoPsIdzhtcHBQ4+PjQdcMXTLYLgh3OC+RSARdM3v27NEf/dEfxVwR0H2EO5z31FNP6cEHH5T0UR98oy83Aa7gh8O2qf/+7//Wn//5n9e9Dtw1NtD/7d/+TSdOnIi5mt44cOCA/vZv/zbuMhATTmG2qZWVFS0sLMRdRs88/vjjeuSRR7bNT/xeu3ZN3/72t+MuAzHizH2bu3r1atwloAvm5+cZON7mOHMHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3LEpxWJRCwsLGhsbi7sUACH8njs25cKFC7p8+XLcZbTM87y6j6XTaY2MjOjgwYPatWtXD6sCOoczd2zKpUuX4i6hLcYYFQqF4H6pVJIxRsYYHTlyRLOzs3rxxRdVLBZjrBJoH+GObWt4eDj4f/gMfXR0VG+88YYk6cyZMyqXyz2vDdgswh0tKZfLWlhYkOd5Ghsb061btyLnKxaLunjxYjDfyspKMD3cR7+0tBTMc/v27ao27PKzs7MqFovrulLqrUOSZmZmNDMz0/Z2Dg8P69VXX9XS0pJu3LjRV9sGNMVgW5qbmzPtPP2+75tkMmlKpZIxxphsNmskVbVVKBSM7/smm80aY4xZXl42kkwulzO+7wfzr66uGmOMyefzRpJJJpNBG+l02uTzeWOMMaVSyaRSqabXYYwxqVTKpFKpDbentvawUqm0rq5+2LZmtPv8whmv8OxvU+28+BcXF40ks7a2FkyzARhuywZ+mKQgbKMCtXaaJFMoFIL7hUKhpXU0q1G4Rz2+VbaNcN/2CPftqp0XfzKZjFymNrzCZ7C1t6j5o6bZdWWz2eBTQthG62hWq+G+VbaNcN/2CPftqp0Xf72AiTozbSUwo6atra1VhVw6nW6qllY10y0TPmPeKttGuG97rzCgiq6pN9jajJGRES0uLiqXyymZTOr8+fO6ePFiR9exkbfffluSdOjQoY6utx+2De4j3NG0TCYjSXrnnXeamu/KlSvBZYT26o9meZ6ncrms0dFRXbp0SblcTufPn+/oOhopFot6/fXX5fu+Dh8+3NH1xr1t2Cbi/uyAeLTzsd1e+eH7fnC1h72SQ6ErQuwAYe0tn89XPWb7m8ODsnagUb/tDrHryefzVd0XjdZhTHNXy4TXG+77tle++L5fNfDZL9vWDLpltj363Lerdl/8+Xw+GBBMJpNVl+2FgzCfzweX+CWTySCYagOr0bRCoWDS6XRkv3SjdRizcbhHhae9pdPp4FLGevsgzm1rBuG+7b3iGWPMJk78sUXNz89rcnJSPP1u4vnd9s7R5w4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOGoy7AMTrxIkTcZeALrh27VrcJSBmhPs2dfjwYY2Pj+vu3btxl9ITxWJRP/vZz3Tw4MG4S+mJ48eP68CBA3GXgRjxN1SxLcPaWwUAAA9PSURBVPA3RbHN8DdUAcBFhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYNxFwB0w5kzZ/STn/xEu3fvliT9+te/1uDgoL74xS8G8/zqV7/Sd7/7XT377LMxVQl0D+EOJ33/+9+PnH79+vWq+zdv3iTc4SS6ZeCkb33rWxoaGtpwvpMnT/agGqD3CHc4aXx8XJVKpeE8jz32mB599NEeVQT0FuEOJz3yyCP67Gc/K8/zIh8fGhrSqVOnelwV0DuEO5w1NTWlgYGByMfu3LmjiYmJHlcE9A7hDmedPHlSd+/eXTd9x44deuKJJ7Rv374YqgJ6g3CHsx566CE9+eST2rGj+jD3PE9TU1MxVQX0BuEOp50+fTqy3/2FF16IoRqgdwh3OO3YsWNV4T4wMKBDhw5peHg4xqqA7iPc4bQHHnhAzzzzTDCwaozR6dOnY64K6D7CHc47deqUjDGSProE8ujRozFXBHQf4Q7nPf/889q5c6ck6bnnntN9990Xc0VA9/HbMg567733dPPmzbjL6Cv79+/XT3/6U+3fv1/Xrl2Lu5y+MTAwoLGxMQ0OEgWu8Yz9vApnvPTSS3rzzTfjLgNbxA9+8AO6qtxzjrdrB33wwQdKJBKam5uLuxT0Oc/z9Jvf/CbuMtAF9LkDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7qirWCxqYWFBY2NjcZcCoEX8njvqunDhgi5fvhx3GW0rl8v6j//4D/37v/+7lpaWtLi42HIbnufVfSydTmtkZEQHDx7Url27NlMq0HGcuaOuS5cuxV3CpqTTaf3TP/2TXn75ZS0tLbXVhjFGhUIhuF8qlWSMkTFGR44c0ezsrF588UUVi8VOlQ10BOEOZ7322mt67bXXNt3O8PBw8P/wGfro6KjeeOMNSdKZM2dULpc3vS6gUwh3BMrlshYWFuR5nsbGxnTr1q3I+YrFoi5evBjMt7KyEkwP99EvLS0F89y+fbuqDbv87OysisXiuu6PeuvotJmZGc3MzLS9/PDwsF599VUtLS3pxo0bVY+5tJ+wBRk4J5FImEQi0fJyvu+bZDJpSqWSMcaYbDZrJJnwYVIoFIzv+yabzRpjjFleXjaSTC6XM77vB/Ovrq4aY4zJ5/NGkkkmk0Eb6XTa5PN5Y4wxpVLJpFKpptfRjtptCEulUiaVSm2qjVKptG4bt8p+kmTm5uaanh9bxiuEu4PaCffFxUUjyaytrQXTbGiFA8UGfpikICCjQrB2miRTKBSC+4VCoaV1tKpRMHeqja26nwh3Z71CtwwkSf/8z/8sSRoZGQmmRV0BMj8/L+mjq0jsTZL++q//uul1JZNJ7dmzRwsLCyqXyxoeHpYxpqPriBv7CbGL++0FndfOmbvqnJnWTq83X6PHa6etra1VdU2k0+mmamlXJ9pr1Ib9hBM+Y94q+0mcubuKM3e0p95gazNGRka0uLioXC6nZDKp8+fP6+LFix1dRy+9/fbbkqRDhw6te4z9hLgQ7pAkZTIZSdI777zT1HxXrlwJLv2zV2w0y/M8lctljY6O6tKlS8rlcjp//nxH19ErxWJRr7/+unzf1+HDh4Pp7CfELu7PDui8drpl7NUavu8HV2jYqy8UuorDDurV3vL5fNVj9oqb8KCsHRzUb7sw7Hry+XxVl0OjdbQqvH5bU1gzV8vUa8Ne+eL7ftXA51baT6JbxlV0y+Aje/fuVT6f10MPPaR9+/Zpenpaf/AHfyDf95XNZvVXf/VXkj66rjufzyuVSkn6aNAvn89r79692rNnT9De7t27q/6VVPX4n/3Zn+natWvyPE/Xrl3TX/zFXwSPNVpHKzzPq1r/7t27G/6cQCtteJ6nf/mXf9Ff/uVfanFxseqLThttQ7/tJ7jJMyY0/A4nTE5OSpLm5uZirgT9zvM8zc3NKZFIxF0KOuscZ+4A4CDCHQAcxE/+Yktpts+c3kZsd4Q7thRCG2gO3TIA4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAO4lchHXXt2jUdPXo07jIAxIRwd9CnP/1pVSoVnThxIu5SsAUcOHAg7hLQBfwNVWwL8/Pzmpyc5PfgsV3wN1QBwEWEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4A4CDCHcAcBDhDgAOItwBwEGEOwA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBg3EXAHTD8vKyfv7znwf333rrLUlSJpOpmu9LX/qS9u7d29PagF7wjDEm7iKATvM8T5I0NDQkSTLGyBijHTvufVitVCr6+te/ru985zux1Ah00Tm6ZeCkl156SUNDQ6pUKqpUKrpz547u3r0b3K9UKpKkQ4cOxVwp0B2EO5w0MTERBHg9999/v44cOdKjioDeItzhpEOHDumTn/xk3ceHhoY0Pj6uwUGGneAmwh1OGhgY0KlTp7Rz587IxyuVihKJRI+rAnqHcIezEomEPvzww8jHHnzwQT311FM9rgjoHcIdzvrc5z6nhx9+eN30oaEhnT59OriiBnAR4Q5neZ6nqamp4HJIq1KpaHx8PKaqgN4g3OG0RCKx7qqZAwcOaHR0NKaKgN4g3OG0Rx99VJ/5zGeC+0NDQ/rqV78aX0FAjxDucN7p06eDrpk7d+5oYmIi5oqA7iPc4byJiQnduXNHkvT4449r//79MVcEdB/hDuft27cv6GOfmpqKuRqgN/jhsD6USqX0N3/zN3GXATTlX//1X/XEE0/EXQaqneO7133oP//zPzU0NKS5ubm4S3HG3bt3VSwW9alPfSruUpxy4sQJvfvuu4R7HyLc+9Tx48d1/PjxuMsAsEXR5w4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIMIdwBwEOEOAA4i3AHAQYQ7ADiIcAcABxHuAOAgwh0AHES4Y8soFotaWFjQ2NhY3KUAfY/fc8c65XJZu3fvVrf+SFe77V+4cEGXL1/uSk3d4Hle3cfS6bRGRkZ08OBB7dq1q4dVYbvgzB3r3Lhxoy/bv3TpUocr6S5jjAqFQnC/VCrJGCNjjI4cOaLZ2Vm9+OKLKhaLMVYJVxHuqFIulzU7O7tl2+83w8PDwf/DZ+ijo6N64403JElnzpxRuVzueW1wG+HukHK5rIWFBXmeJ8/zNDs7W3VWaKeHuwtqp6XTaS0tLVU9ViwWtbS0FPR1z87OyvM8TU9P69atW5tuf7PbbOvxPE8zMzMqFou6ePFi1bovXrwYLBN+7Pbt25JUtczY2JhWVlaC6Xbby+WypqenNTMzI0mamZkJ/t+O4eFhvfrqq1paWlr3aaZRPeFxh6WlpWAeuy2122mPg9p9XW8dcIRB30kkEiaRSLS8nO/7JpPJGGOMKRQKxvd94/u+KZVKwTRJJvy05/P5ddPq3ZdkVldXjTHGlEolk0wmjSSztra2qfZbUbusraFQKATrSiaTxhhjVldXq+7X7qtCoVC1r7LZrDHGmOXlZSPJ5HI54/t+1bbncrmgvVQqZVKpVMs1h5VKpXU1tlKPMWbddhtjTDqdNvl8PlhHKpWqqqHROlohyczNzbW0DHriFcK9D7UT7vbFaQPLmHvhZl/AxkQHTTPhGzUtl8sZSSadTm+6/WbVLptKpapCrfbxdDptJAVBZ+sO75NsNhtZsw1u26Z9k9xszRs93mw9jdqoPRbsG2+z62gW4d63CPd+1E642zPYMHtW6Pt+MK2T4d7usp0MdyufzwdBHn7cvgHZTzTGVJ/VGmOqzoZrb5utt5nlax9vp556n2iy2Wzkm9JG62hl2wj3vkS496N2wr3b4dvP4Z7JZIzv+2ZtbS3ycRt0pVIp6E5qZts6Ue9Gy9s34PAZczv11E5bW1urCvDwp6tm1tEswr1vvcKAqiN835ekyMvqkslkV9fd7fYbWVhY0Msvv6zvfe97GhkZiZzH1vfDH/5QN27c0NTUVOR84cHhXnn77bclSYcOHVr32GbqGRkZ0eLionK5nJLJpM6fP181qNyJdaC/Ee6OSCQSkqRf/OIXwTR7ed3x48e7sk4bDF/+8pe70n4zJiYmJEl79+6tO8/o6KiSyaQmJiY0Ozurz3/+81WPZzIZSdKVK1eCfWavJOmmYrGo119/Xb7v6/Dhwx2tx/M8lctljY6O6tKlS8rlcjp//nxH14E+F/dnB6zXTrdMqVQKro6xA2nZbHZdF0TtFS520FW6d7WF/ThfKBSCj/N2HjsQaa/ACPfnb6b9ZoSvxrHbaNvK5/NV3TLhwcRwHeG+96h2w7d8Ph95BZDVzNUytttFqh6QtVe+hJ+vVuux7YXXYdvSb7t67NiCHZNoZh2tEN0y/Yo+937U7qWQhULBZDKZqiCuHUzL5/NBIC4uLhpjTHBJnA0GOwiZSqWqwkKqvhwvk8l0rP1m1AZRVFv26pmokLL98lHy+XxwuWB4+fD6at/INgr3qPC0t3Q6HVzK2G49dh/Um2bfPO36ml1HKwj3vvWKZ0yXfkAEbZucnJQkzc3NxVzJPfYLMFv1cCmXy/rGN76x5X7CoN95nqe5ubmgWxB94xx97tgWrl692rWxB6AfEe7YUPgKnK30I1czMzNVPzMQHrQEXMdP/mJDe/bsqfp/p7tmmv19mVbXa6+gyWQyOnv2bMt1AVsZ4Y4NdbufvVvtnz17llDHtkW3DAA4iHAHAAcR7gDgIMIdABxEuAOAgwh3AHAQ4Q4ADiLcAcBBhDsAOIhwBwAHEe4A4CDCHQAcRLgDgIP4Vcg+9LGPfUxvvvmm5ufn4y4F2NAnPvGJuEtABP7MXh967733dPPmzbjLADY0MDCgsbExDQ5ynthnzhHuAOAe/oYqALiIcAcABxHuAOCgQUlfi7sIAEBH/ej/AeTMH1DamjtZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfk.utils.plot_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=tfk.optimizers.Adam(learning_rate=1e-5),\n",
    "              loss=tfk.losses.CategoricalCrossentropy(label_smoothing=0.1),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tfk.models.load_model('.model_files/saves/model_5.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "741 82\n"
     ]
    }
   ],
   "source": [
    "class_weights = class_weight.compute_class_weight('balanced',\n",
    "                                                  np.unique(train_generator.classes),\n",
    "                                                  train_generator.classes)\n",
    "class_weights = {i : class_weights[i] for i in range(len(class_weights))}\n",
    "\n",
    "train_step_size = train_generator.n//train_generator.batch_size\n",
    "val_step_size   = val_generator.n//val_generator.batch_size\n",
    "print(train_step_size, val_step_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(\n",
    "    monitor='val_accuracy',\n",
    "    factor=0.2,\n",
    "    patience=5,\n",
    "    min_lr=1e-7\n",
    ")\n",
    "\n",
    "early_stop = tfk.callbacks.EarlyStopping(\n",
    "    monitor='val_accuracy',\n",
    "    patience=15,\n",
    "    restore_best_weights=True\n",
    ")\n",
    "\n",
    "checkpoint_dir = os.path.join('.model_files')\n",
    "if not os.path.exists(checkpoint_dir):\n",
    "    os.makedirs(checkpoint_dir)\n",
    "checkpoint = tfk.callbacks.ModelCheckpoint(\n",
    "    filepath = checkpoint_dir + '/ckpt/',\n",
    "    save_freq ='epoch',\n",
    "    save_best_only=True\n",
    ")\n",
    "\n",
    "log_dir = os.path.join(\".model_files/logs/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "if not os.path.exists(log_dir):\n",
    "    os.makedirs(log_dir)\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(\n",
    "    log_dir=log_dir,\n",
    "    histogram_freq=1,\n",
    "    update_freq=100,\n",
    "    profile_batch = 100000000\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 75/100\n",
      "200/200 [==============================] - ETA: 0s - loss: 1.3116 - accuracy: 0.7876INFO:tensorflow:Assets written to: .model_files/ckpt/assets\n",
      "200/200 [==============================] - 863s 4s/step - loss: 1.3116 - accuracy: 0.7876 - val_loss: 1.4584 - val_accuracy: 0.7531 - lr: 1.0000e-05\n",
      "Epoch 76/100\n",
      "200/200 [==============================] - ETA: 0s - loss: 1.3194 - accuracy: 0.7832INFO:tensorflow:Assets written to: .model_files/ckpt/assets\n",
      "200/200 [==============================] - 820s 4s/step - loss: 1.3194 - accuracy: 0.7832 - val_loss: 1.4416 - val_accuracy: 0.7625 - lr: 1.0000e-05\n",
      "Epoch 77/100\n",
      "200/200 [==============================] - 801s 4s/step - loss: 1.3097 - accuracy: 0.7858 - val_loss: 1.4463 - val_accuracy: 0.7576 - lr: 1.0000e-05\n",
      "Epoch 78/100\n",
      "200/200 [==============================] - ETA: 0s - loss: 1.3048 - accuracy: 0.7890INFO:tensorflow:Assets written to: .model_files/ckpt/assets\n",
      "200/200 [==============================] - 900s 4s/step - loss: 1.3048 - accuracy: 0.7890 - val_loss: 1.4330 - val_accuracy: 0.7625 - lr: 1.0000e-05\n",
      "Epoch 79/100\n",
      "200/200 [==============================] - 927s 5s/step - loss: 1.3037 - accuracy: 0.7907 - val_loss: 1.4581 - val_accuracy: 0.7543 - lr: 1.0000e-05\n",
      "Epoch 80/100\n",
      "200/200 [==============================] - 857s 4s/step - loss: 1.3108 - accuracy: 0.7871 - val_loss: 1.4504 - val_accuracy: 0.7539 - lr: 1.0000e-05\n",
      "Epoch 81/100\n",
      "200/200 [==============================] - 783s 4s/step - loss: 1.3064 - accuracy: 0.7861 - val_loss: 1.4820 - val_accuracy: 0.7487 - lr: 1.0000e-05\n",
      "Epoch 82/100\n",
      "200/200 [==============================] - 651s 3s/step - loss: 1.3081 - accuracy: 0.7891 - val_loss: 1.4781 - val_accuracy: 0.7402 - lr: 2.0000e-06\n",
      "Epoch 83/100\n",
      "200/200 [==============================] - 665s 3s/step - loss: 1.3078 - accuracy: 0.7875 - val_loss: 1.4561 - val_accuracy: 0.7508 - lr: 2.0000e-06\n",
      "Epoch 84/100\n",
      "200/200 [==============================] - ETA: 0s - loss: 1.3062 - accuracy: 0.7880INFO:tensorflow:Assets written to: .model_files/ckpt/assets\n",
      "200/200 [==============================] - 778s 4s/step - loss: 1.3062 - accuracy: 0.7880 - val_loss: 1.4276 - val_accuracy: 0.7563 - lr: 2.0000e-06\n",
      "Epoch 85/100\n",
      "200/200 [==============================] - 735s 4s/step - loss: 1.3081 - accuracy: 0.7883 - val_loss: 1.4308 - val_accuracy: 0.7662 - lr: 2.0000e-06\n",
      "Epoch 86/100\n",
      "200/200 [==============================] - 751s 4s/step - loss: 1.3082 - accuracy: 0.7843 - val_loss: 1.4465 - val_accuracy: 0.7527 - lr: 2.0000e-06\n",
      "Epoch 87/100\n",
      "200/200 [==============================] - 819s 4s/step - loss: 1.3143 - accuracy: 0.7838 - val_loss: 1.4358 - val_accuracy: 0.7598 - lr: 2.0000e-06\n",
      "Epoch 88/100\n",
      "200/200 [==============================] - 819s 4s/step - loss: 1.2907 - accuracy: 0.7923 - val_loss: 1.4625 - val_accuracy: 0.7516 - lr: 2.0000e-06\n",
      "Epoch 89/100\n",
      "200/200 [==============================] - 778s 4s/step - loss: 1.2996 - accuracy: 0.7901 - val_loss: 1.4642 - val_accuracy: 0.7515 - lr: 2.0000e-06\n",
      "Epoch 90/100\n",
      "200/200 [==============================] - 787s 4s/step - loss: 1.3090 - accuracy: 0.7876 - val_loss: 1.4642 - val_accuracy: 0.7449 - lr: 2.0000e-06\n",
      "Epoch 91/100\n",
      "200/200 [==============================] - 803s 4s/step - loss: 1.3029 - accuracy: 0.7904 - val_loss: 1.4836 - val_accuracy: 0.7473 - lr: 4.0000e-07\n",
      "Epoch 92/100\n",
      " 33/200 [===>..........................] - ETA: 9:36 - loss: 1.3097 - accuracy: 0.7862"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-45-45ef13f3ad9d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m                               \u001b[0mearly_stop\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m                               \u001b[0mcheckpoint\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m                               \u001b[0mtensorboard_callback\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m                               ]\n\u001b[0;32m     13\u001b[0m                    )\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m     64\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 66\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     67\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m     \u001b[1;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[0;32m    846\u001b[0m                 batch_size=batch_size):\n\u001b[0;32m    847\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 848\u001b[1;33m               \u001b[0mtmp_logs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    849\u001b[0m               \u001b[1;31m# Catch OutOfRangeError for Datasets of unknown size.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    850\u001b[0m               \u001b[1;31m# This blocks until the batch has finished executing.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    578\u001b[0m         \u001b[0mxla_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mExit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    579\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 580\u001b[1;33m       \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    581\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    582\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mtracing_count\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\def_function.py\u001b[0m in \u001b[0;36m_call\u001b[1;34m(self, *args, **kwds)\u001b[0m\n\u001b[0;32m    609\u001b[0m       \u001b[1;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    610\u001b[0m       \u001b[1;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 611\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=not-callable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    612\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    613\u001b[0m       \u001b[1;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2418\u001b[0m     \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2419\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2420\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# pylint: disable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2421\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2422\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[1;34m(self, args, kwargs)\u001b[0m\n\u001b[0;32m   1663\u001b[0m          if isinstance(t, (ops.Tensor,\n\u001b[0;32m   1664\u001b[0m                            resource_variable_ops.BaseResourceVariable))),\n\u001b[1;32m-> 1665\u001b[1;33m         self.captured_inputs)\n\u001b[0m\u001b[0;32m   1666\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1667\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[1;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[0;32m   1744\u001b[0m       \u001b[1;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[1;32m-> 1746\u001b[1;33m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[0;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[0;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\function.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[0;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 598\u001b[1;33m               ctx=ctx)\n\u001b[0m\u001b[0;32m    599\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[1;32mc:\\users\\franz cesista\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\eager\\execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[1;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[0;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[1;32m---> 60\u001b[1;33m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[0;32m     61\u001b[0m   \u001b[1;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "history = model.fit(train_generator,\n",
    "                    initial_epoch=74,\n",
    "                    epochs=100,\n",
    "                    steps_per_epoch=200,\n",
    "                    validation_steps=20,\n",
    "                    validation_data=val_generator,\n",
    "                    class_weight=class_weights,\n",
    "                    callbacks=[reduce_lr,\n",
    "                              early_stop, \n",
    "                              checkpoint, \n",
    "                              tensorboard_callback\n",
    "                              ]\n",
    "                   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('model_5.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tfk.models.load_model('.model_files/saves/model_1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "      <th>dir_filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30694</th>\n",
       "      <td>0000ffaa6ce77e3dde38bcefa039a4a4.jpg</td>\n",
       "      <td>16</td>\n",
       "      <td>16/0000ffaa6ce77e3dde38bcefa039a4a4.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50947</th>\n",
       "      <td>00017f8bf9d8a34dab8f9707a378dbe5.jpg</td>\n",
       "      <td>38</td>\n",
       "      <td>38/00017f8bf9d8a34dab8f9707a378dbe5.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31622</th>\n",
       "      <td>00022314f9c58eb20d248ed239db3a52.jpg</td>\n",
       "      <td>16</td>\n",
       "      <td>16/00022314f9c58eb20d248ed239db3a52.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34234</th>\n",
       "      <td>0002d098348d8358c44032903e117ade.jpg</td>\n",
       "      <td>28</td>\n",
       "      <td>28/0002d098348d8358c44032903e117ade.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12787</th>\n",
       "      <td>000367a6ec58e9d1e4bc340d8d871dab.jpg</td>\n",
       "      <td>34</td>\n",
       "      <td>34/000367a6ec58e9d1e4bc340d8d871dab.jpg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   filename category  \\\n",
       "30694  0000ffaa6ce77e3dde38bcefa039a4a4.jpg       16   \n",
       "50947  00017f8bf9d8a34dab8f9707a378dbe5.jpg       38   \n",
       "31622  00022314f9c58eb20d248ed239db3a52.jpg       16   \n",
       "34234  0002d098348d8358c44032903e117ade.jpg       28   \n",
       "12787  000367a6ec58e9d1e4bc340d8d871dab.jpg       34   \n",
       "\n",
       "                                  dir_filename  \n",
       "30694  16/0000ffaa6ce77e3dde38bcefa039a4a4.jpg  \n",
       "50947  38/00017f8bf9d8a34dab8f9707a378dbe5.jpg  \n",
       "31622  16/00022314f9c58eb20d248ed239db3a52.jpg  \n",
       "34234  28/0002d098348d8358c44032903e117ade.jpg  \n",
       "12787  34/000367a6ec58e9d1e4bc340d8d871dab.jpg  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_train_clean = pd.read_csv('shopee-product-detection-dataset/train.csv').sort_values(by=['filename'])\n",
    "pd_train_clean['dir_filename'] = pd_train_clean.apply(lambda x: '{:02d}/{}'.format(x.category, x.filename), axis=1)\n",
    "pd_train_clean['category'] = pd_train_clean['category'].apply(lambda x : '{:02d}'.format(x))\n",
    "pd_train_clean.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 105392 validated image filenames belonging to 42 classes.\n"
     ]
    }
   ],
   "source": [
    "clean_datagen = tfk_ImageDataGenerator(\n",
    "    preprocessing_function=tfk_preprocess_input,\n",
    ")\n",
    "\n",
    "clean_generator = clean_datagen.flow_from_dataframe(\n",
    "    pd_train_clean,\n",
    "    directory='shopee-product-detection-dataset/train/',\n",
    "    x_col='dir_filename',\n",
    "    y_col='category',\n",
    "    target_size=input_shape[:2],\n",
    "    color_mode='rgb',\n",
    "    batch_size=batch_size,\n",
    "    class_mode='categorical',\n",
    "    shuffle=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Model was constructed with shape (None, 299, 299, 3) for input Tensor(\"xception_input:0\", shape=(None, 299, 299, 3), dtype=float32), but it was called on an input with incompatible shape (32, 256, 256, 3).\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 299, 299, 3) for input Tensor(\"input_1:0\", shape=(None, 299, 299, 3), dtype=float32), but it was called on an input with incompatible shape (32, 256, 256, 3).\n",
      "step: 10\n",
      "step: 20\n",
      "step: 30\n",
      "step: 40\n",
      "step: 50\n",
      "Wall time: 1min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_pred = []\n",
    "y_true = []\n",
    "\n",
    "for i in range(50):\n",
    "    if (i+1) % 10 == 0:\n",
    "        print(\"step:\", i+1)\n",
    "    x, y = next(iter(val_generator))\n",
    "    y_true.extend(y)\n",
    "    y_pred.extend(model.predict(x))\n",
    "\n",
    "y_pred = np.array(y_pred)\n",
    "y_true = np.array(y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix = confusion_matrix(y_true.argmax(axis=1), y_pred.argmax(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA08AAAILCAYAAAAwgKfUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzde3RV9Z3//1ckBKEM7aiJWKBQKQz+rEK9DEYt0Y6NQBIxoDbRAuEWsDFeoARywdMDniRcaqxNbRtMQcFbVIKABkXjUGMUauoERUvTahgDGYpfZqQZIDHJ/v3BIlO8nZx9yfkkeT7W2muds0/2a7/3ziH6Xp+9PzvCsixLAAAAAICvdEa4CwAAAACA7oDmCQAAAAA6geYJAAAAADqB5gkAAAAAOoHmCQAAAAA6geYJAAAAADohMhw79fv94dgtAAAAYDSfzxfuEkL26ccfeJrf95zzPc0PRUQ4nvMUGTXE0fbLctMlSSsCJUZkDC+ps52xP32Ua3W4kXFw3bu2M74567uSpK+V2j8f/zvHrPPRkzLuX/2o7YyFi2e4VkdPyDiVQ4b7GYH8tY4ycnPmGXMspmQ4+bcvnfz3z++FjO6QMe2RPzjKeHbm5UYcS2vLAUfbh0Nvap4cXba3detWTZ48WfHx8XrsscfcqgkAAABAd9He5u1iENuX7R06dEhFRUXatGmToqKilJKSovHjx+s73/mOm/UBAAAAgBFsjzxVV1friiuu0De+8Q0NGDBA119/vbZv3+5mbQAAAABMZ7V7uxjEdvP0t7/9TdHR0R3vY2JidOjQIVeKAgAAAADT2L5sr729XRERER3vLcs67T0AAACAXqDdrNEhL9keeRo8eLAOHz7c8f7w4cOKiYlxpSgAAAAAMI3t5unKK6/UG2+8oSNHjuj48eN66aWXNGHCBDdrAwAAAGA4y2r3dDGJ7cv2zj33XN1zzz2aMWOGPv30U9100026+OKL3awNAAAAAIzh6DlPSUlJ2rZtm1588UXNmzfPrZoAAAAAdBft7d4uIWpqalJiYqIaGhoknZwlPCkpSfHx8SoqKur4uffff19Tp07V9ddfr9zcXLW2tgbNjrAsywq5Iof8fn9X7xIAAAAwns/nC3cJIWtpeMfT/KihF3X6Z2tra5WXl6cPP/xQ27dv1znnnKOJEydqw4YNOu+88zR//nzNmDFDcXFxSkxM1H333adx48YpJydH3/3ud3Xrrbd+Zb7ty/acWBEocbT9stx0xzluZrSs22c7I2rWv7hWhxsZg0rrbGccnTNKkvTJuvdtZ3x91gWSzDkfZJDxVTlkuJ9x/+pHHWUsXDzDmGMxJSOQv9ZRRm7OPH4vZHie4cb39J7/rHKUUfStq404H92xeTLpWUxlZWXy+XzKysqSJO3Zs0fDhw/XsGHDJJ28cm779u36zne+oxMnTmjcuHGSpKlTp+rBBx80s3kCAAAAgM44evSojh49+rn1gwYN0qBBg05bFwgETnv/Zc+m/ez66OjoTj2z1nHz1NTUpJSUFP3mN7/R0KFDncYBAAAA6E7a2zyNf+SRR1RcXPy59XfccYcyMzO/ctsvezat3WfWOmqeTl1TWF9f7yQGAAAAAL7QzJkzlZyc/Ln1nx11+iJf9mzaz67/+OOPO/XMWkez7Z26ppCH4wIAAAC9lNXu6TJo0CANHTr0c0tnmqexY8fqww8/1P79+9XW1qZt27ZpwoQJGjJkiPr166eamhpJ0nPPPdepZ9Y6Gnn67DWFAAAAAGCKfv36qbCwUJmZmWpublZcXJwmTpwoSVqzZo3y8vLU1NSkCy+8UDNmzAiax4QRAAAAAOyz8Swmr1VWVna8jo2N1ZYtWz73M2PGjNEzzzwTUq6jy/YAAAAAoLdg5AkAAACAbZZBz3nyGiNPAAAAANAJrow8/eM1hQAAAAB6EQPvefJKhGVZVlfv1O/3d/UuAQAAAOP5fL5wlxCy5j9XeZrfb/TVnuaHIizNU2TUEEfbL8tNlyStCJQYkXFk3fu2M86adYFrdbiRcXDdu7Yzvjnru5KkTxycj68bdj56UsbqlettZyxekuZaHT0h41QOGe5nFBSUOsrIzp5jzLGYkuHk37508t9/IH+to4zcnHnGnA8yem5G5h+dXQn1y0t+YMSxtLYccLR9OPSm5snRZXvFxcWqqKiQJMXFxSkrK8uVogAAAAB0E+1t4a6gy9ieMKK6ulpVVVUqLy/X5s2btXfvXu3YscPN2gAAAADAGLZHnqKjo7V06VJFRUVJkkaOHKmDBw+6VhgAAACAbqAXTVVuu3kaNWpUx+v6+npVVFToiSeecKUoAAAAADCN46nK6+rqNH/+fGVlZWnEiBEulAQAAACg2+hFU5U7ekhuTU2N0tLStGjRIiUnJ7tVEwAAAAAYx/bIU2NjozIyMlRUVKTY2Fg3awIAAADQXXDPU3ClpaVqbm5WYWFhx7qUlBSlpqa6UhgAAAAAmMR285SXl6e8vDw3awEAAADQ3fSie54iLMuyunqnfr+/q3cJAAAAGM/n84W7hJA173nR0/x+F1/vaX4oHM+2BwAAAKD3sqy2cJfQZcIy8hQZNcTR9sty0yVJKwIljjMKCkptZ2Rnz5EkHVv3J9sZA2aNkeTOsbiR8cm6921nfH3WBZKkqN/V2c5omX3y+WGmnA9TMgL5a21n5ObMkyStXrnedsbiJWmSzDkf4c44lUOG+xlOvqfSye+qKcdiSoaTvx/Syb8hbmSYcj7I6LkZc7fvdJTx8MQ4I46lteWAo+3D4UTtC57mnzl2sqf5oXA08vSLX/xCL774oiIiInTTTTdp1qxZbtUFAAAAoDtgtr3gdu/erTfffFNbtmxRa2urJk+erLi4OJ1//vlu1gcAAAAARrDdPP3rv/6rHn30UUVGRurQoUNqa2vTgAED3KwNAAAAgOl60Wx7ZzjZuG/fvnrwwQeVkJCg2NhYnXvuuW7VBQAAAABGcdQ8SdKdd96pN954Q42NjSorK3OjJgAAAADdhdXu7WIQ283TX//6V73//smZ2fr376/4+Hjt27fPtcIAAAAAwCS2m6eGhgbl5eWppaVFLS0teuWVV3TppZe6WRsAAAAA07W3ebsYxPaEEXFxcdqzZ49uvPFG9enTR/Hx8UpISHCzNgAAAAAwhqPnPGVmZiozM9OtWgAAAAB0N4bdl+SlCMuyrK7eqd/v7+pdAgAAAMbz+XzhLiFkJ958ytP8M6/4kaf5oQhL8xQZNcTR9sty0yVJKwIlRmSkPvmG7YwnUmJdq8ONjP3r3rGdMXzWRZKkI+vet51x1qwLJJlzPsgg46tyTMkI5K91lJGbM49j+UwdPSmjoKDUUUZ29hytXrneUcbiJWnGnA8yzMxw43s67ZE/OMp4dublRpyP1pYDjrYPh97UPDm6bA8AAABAL9eLLttz/JwnSVq5cqWWLl3qRhQAAAAAGMlx8/TGG2+ovLzcjVoAAAAAdDft7d4uBnHUPP3P//yPioqKtGDBArfqAQAAAAAjObrn6d5779U999yjxsZGt+oBAAAA0J0YNjrkJdsjT08//bTOO+88xcbGulkPAAAAABjJ9sjTCy+8oMOHD2vKlCn65JNPdOzYMeXn5ysnJ8fN+gAAAAAYzLLawl1Cl7HdPK1bt67j9aZNm7R7924aJwAAAAA9Fs95AgAAAGBfL7rnyZXmaerUqZo6daobUQAAAABgpAjLsqyu3qnf7+/qXQIAAADG8/l84S4hZMdffdjT/P7XzvU0PxRhaZ4io4Y42n5ZbrokaUWgxIiMl0p22s6IT49zrQ43MmZt/b3tjHVJEyRJv1/7uu2MCfOukmTO+ehJGQUFpbYzsrPnuFZHT8g4lUOG+xmB/LWOMnJz5hlzLKZkcE7J6A4ZTv4bJZ3879S0R/7gKOPZmZcbcT5aWw442j4celPz5OiyvenTp+vIkSOKjDwZs3z5co0dO9aVwgAAAAB0A9zzFJxlWaqvr9err77a0TwBAAAAQE9l+yG5H3zwgSRp9uzZuuGGG7Rx40bXigIAAADQTVjt3i4GsT1kdPToUcXGxmrZsmX69NNPNWPGDH3729/WVVdd5WZ9AAAAAGAE283T9773PX3ve9/reH/TTTdp586dNE8AAABAb9KL7nmyfdneW2+9pTfeeKPjvWVZ3PsEAAAAoMey3Tz9/e9/16pVq9Tc3KympiaVl5frhz/8oZu1AQAAADAd9zwFd+2116q2tlY33nij2tvbdeutt552GR8AAACAXqAXXbbn6Dq7u+++W3fffbdbtQAAAACAsSIsy7K6eqd+v7+rdwkAAAAYz+fzhbuEkB1//gFP8/snmDNYE5bmKTJqiKPtl+WmS5JWBEqMyHipZKftjPj0ONfqcCPj+w/ttZ3x2k8ulCQ9/VCF7YybfzJJkjnnoydlFBSU2s7Izp7jWh09IeNUDhnuZ9y/+lFHGQsXzzDmWEzJcPJvXzr579+UY3EjI5C/1lFGbs48Y46FjNMzflJV6Sjjoat/YMR3rKW5wdH24dCbmiemxwMAAABgn2GTOnjJ9mx7klRZWampU6dq0qRJuu+++9yqCQAAAACMY7t5+uijj+Tz+fTQQw9py5Yteu+997Rzp/3L1wAAAAB0Q+3t3i4GsX3Z3o4dOzR58mQNHjxYklRUVKR+/fq5VhgAAAAAmMR287R//3717dtXCxYsUGNjo6655hqmLQcAAAB6G+55Cq6trU1vvPGG8vPz9dRTT2nPnj0qLy93szYAAAAAMIbt5umcc85RbGyszjrrLJ155pm67rrrtGfPHjdrAwAAAGC6XnTPk+3m6dprr1VVVZWOHj2qtrY2vfbaa7rwwgvdrA0AAAAAjGH7nqexY8dq7ty5uvXWW/Xpp5/qqquu0rRp09ysDQAAAIDpetE9T44eknvTTTfppptucqsWAAAAADBWhGVZVlfv1O/3d/UuAQAAAOP5fL5wlxCy48/c52l+/5vyPM0PRViap8ioIY62X5abLklaESgxImPrb162nZG04DrX6nAj44+lf7CdccmcyyVJ+9e9Yztj+KyLJJlzPnpSRiB/re2M3Jx5rtXREzJO5ZBxeoaT75h08ntWUFDqKCM7e44x58OUjPtXP+ooY+HiGVq9cr2jjMVL0ow5H2T03IyfRn3gKGNNy/lGHEtrywFH24dDb2qeHF22BwAAAKCXM2xGPC/Zbp6efvppbdy4seN9Q0ODpkyZonvvvdeVwgAAAADAJLabp5tvvlk333yzJKmurk4ZGRm64447XCsMAAAAQDfQ9XcBhY3t5zz9o5/97Ge65557dNZZZ7kRBwAAAADGcXzPU3V1tU6cOKFJkya5UQ8AAACA7oR7njrvySef1KxZs9yoBQAAAEB304uaJ0eX7bW0tOgPf/iDfvCDH7hVDwAAAAAYydHI0759+zRixAgNGDDArXoAAAAAdCcWI0+d8tFHH2nw4MFu1QIAAAAAxnI08jR58mRNnjzZrVoAAAAAdDe96J6nCMvq+onZ/X5/V+8SAAAAMJ7P5wt3CSE7/mi2p/n9ZxR4mh+KsDRPkVFDHG2/LDddkrQiUGJExqtrX7Odce2877tWhxsZU35XYzvjudmXSpIOrnvXdsY3Z31XkjnngwwyviqHDPczAvlrHWXk5swz5lhMybh/9aOOMhYunmHMsZBBxldl/DTiz44y1lijjTiW1pYDjrYPh+OPLPU0v//MQk/zQ+HonqfnnntOCQkJSkhI0MqVK92qCQAAAACMY/uep+PHjysQCGj79u0aNGiQUlNTVV1drSuvvNLN+gAAAACYrBfd82R75KmtrU3t7e06fvy4Wltb1draqn79+rlZGwAAAAAYw/bI08CBA3XXXXdp0qRJ6t+/vy6//HJdcsklbtYGAAAAwHSMPAX3pz/9Sc8++6xeffVVvfbaazrjjDNUWlrqZm0AAAAAYAzbzVNVVZViY2N19tlnKyoqSlOnTtXu3bvdrA0AAACA6ax2bxeD2G6exowZo+rqah07dkyWZamyslIXXXSRm7UBAAAAgDFs3/N09dVX67333tPUqVPVt29fXXTRRUpPT3ezNgAAAACGs9q7/LGxYWO7eZKk9PR0GiYAAAAAvUKEZVld3ir6/f6u3iUAAABgPJ/PF+4SQnbsN3d5mj9gwS88zQ9FWJqnyKghjrZflntytGtFoMSIjM2/fsl2xo23x7tWhxsZZz9cZzvj/80dJUnaVbrLdsb4OeMlmXM+yCDjq3JMyQjkr3WUkZszj2P5TB09KeP+1Y86yli4eIZWr1zvKGPxkjRjzgcZPTcja1x/Rxmr/uO4EcfS2nLA0fbhYFLz9Nxzz6mk5OTvYMKECVqyZImqq6tVUFCg5uZmTZo0Sffcc4/tWhxdtgcAAACglzNkRrzjx48rEAho+/btGjRokFJTU1VZWanly5drw4YNOu+88zR//nzt3LlTcXFxtvZhe7Y9SSopKdH111+vpKQk/frXv3YSBQAAAAC2tbW1qb29XcePH1dra6taW1s1cOBADR8+XMOGDVNkZKSSkpK0fft22/uwPfJUXV2trVu36tlnn1X//v2VkZGhl156SfHx8baLAQAAANDNeDzb3tGjR3X06NHPrR80aJAGDRrU8X7gwIG66667NGnSJPXv31+XX365/va3vyk6OrrjZ2JiYnTo0CHbtdhunt577z1dffXVGjhwoCTp+9//vl5++WWaJwAAAKA3aff2sr1HHnlExcXFn1t/xx13KDMzs+P9n/70Jz377LN69dVX9U//9E/66U9/qvr6ekVERHT8jGVZp70Ple3m6cILL1R+fr7mz5+v/v37q7KyUmGYewIAAABADzZz5kwlJyd/bv0/jjpJUlVVlWJjY3X22WdLkqZOnarS0lL16dOn42cOHz6smJgY27XYbp5iY2M1depUTZ8+Xd/4xjcUGxur2tpa24UAAAAA6IY8Hnn67OV5X2bMmDFavXq1jh071jG4M3bsWG3dulX79+/X0KFDtW3bNk2bNs12Lbabp6amJsXHx2vWrFmSpIcffljDhg2zXQgAAAAA2HX11Vfrvffe09SpU9W3b19ddNFFyszM1FVXXaXMzEw1NzcrLi5OEydOtL0P281TQ0ODlixZomeffVbHjx/XM888o0AgYLsQAAAAAN2QQbfupKenKz09/bR1sbGx2rJliyv5tpunMWPGKD4+XjfccIPa2tqUlpamSy+91JWiAAAAAMA0jh6Sm5GRoYyMDLdqAQAAANDdeHzPk0kirDBMkef3+7t6lwAAAIDxfD5fuEsI2bH753maP2DhWk/zQxGW5ikyaoij7ZflnryOcUWgxIiMlnX7bGdEzfoX1+pwI6P64TdtZ1w59wpJ0qDSOtsZR+eMkmTO+SCDjK/KIcP9jIKCUkcZ2dlzjDkWUzLuX/2oo4yFi2do9cr1jjIWL0kz5nyQYWZGIN/Z/xzn5szT4gH7HWWsPjbciPPR2nLA0fbhcGzNXE/zB/z0YU/zQ3FGuAsAAAAAgO6gU81TU1OTEhMT1dDQIEmqrq5WUlKS4uPjVVRU5GmBAAAAAAxmtXu7GCRo81RbW6vU1FTV19dLkk6cOKGcnBw99NBDeuGFF/Tuu+9q586dXtcJAAAAAGEVtHkqKyuTz+dTTEyMJGnPnj0aPny4hg0bpsjISCUlJWn79u2eFwoAAADAQO2Wt4tBgk5V/tkH3/7tb39TdHR0x/uYmBgdOnTI/coAAAAAwCAhP+epvb1dERERHe8tyzrtPQAAAIDew+pFz3kKeba9wYMH6/Dhwx3vDx8+3HFJHwAAAAD0VCE3T2PHjtWHH36o/fv3q62tTdu2bdOECRO8qA0AAACA6bjn6cv169dPhYWFyszMVHNzs+Li4jRx4kQvagMAAAAAY3S6eaqsrOx4HRsbqy1btnhSEAAAAIBuxLBnMXkpwrKsLh8L8/v9Xb1LAAAAwHg+ny/cJYTsf+/7saf5X8vb6Gl+KMLSPEVGDXG0/bLcdEnSikCJERmP/HKz7YyZmTe6VocbGS+V2H/gcXx6nCRp3+/+w3bGv8weJ8mc89GTMgoKSm1nZGfPca2OnpBxKocM9zMC+WsdZeTmzDPmWHpSBr8XMrpDxk+qKoP/4Fd46OofGHEsrS0HHG0fDv+7/DZP879272Oe5oei0xNGNDU1KTExUQ0NDR3rsrKytGnTJk8KAwAAAACTdKp5qq2tVWpqqurr6yVJhw4d0oIFC/Tiiy96WRsAAAAA07W3e7sYpFPNU1lZmXw+X8fznLZu3ap/+7d/06RJkzwtDgAAAIDhmKr8dIFA4LT3c+fOlSTV1NS4XxEAAAAAGCjk5zwBAAAAQIdeNFV5pyeMAAAAAIDejJEnAAAAAPYZdl+Slxh5AgAAAIBOCGnkqbLy9IePFRYWuloMAAAAgO7FMmw6cS9FWJbV5eNsfr+/q3cJAAAAGM/n84W7hJA1ZU/zNH9gwbOe5ociLM1TZNQQR9svy02XJK0IlBiR8cm6921nfH3WBa7V4UZG9cNv2s64cu4VkqRj6/5kO2PArDGSzDkfPSmjoKDUdkZ29hzX6ugJGadyyHA/w8n3VDr5XTXlWEzJCOSvdZSRmzPPlQxTzgcZPTdj8dcPOspY/ck3jTiW1pYDjrYPh6YlUz3NH7hyk6f5oeCeJwAAAADohE41T01NTUpMTFRDQ4Mk6amnnlJiYqKSkpKUnZ2tlpYWT4sEAAAAYKh2y9vFIEGbp9raWqWmpqq+vl6S9OGHH6q0tFRPPvmktmzZovb2dj3++ONe1wkAAAAAYRW0eSorK5PP51NMTIwkKSoqSj6fTwMHDlRERIRGjx6tgwedXWMKAAAAoJuy2r1dDBJ0qvJAIHDa+yFDhmjIkJMTPhw5ckSPPfaYCgoKvKkOAAAAAAwR0nOe/tGhQ4c0d+5cTZs2TePHj3ezJgAAAADdhWH3JXnJ1mx7f/3rX5WSkqLk5GRlZGS4XRMAAAAAGCfkkaempibNmTNHd999t2688UYvagIAAADQTViMPH25Z555Rh9//LHWrVunKVOmaMqUKfrFL37hRW0AAAAAYIxOjzxVVlZKktLS0pSWluZVPQAAAAC6k1408hRhWVaXH63f7+/qXQIAAADG8/l84S4hZH+/M9HT/H96cJun+aEIS/MUGTXE0fbLctMlSSsCJUZkFBSU2s7Izp7jWh1uZMza+nvbGeuSJkiSSh4os52Rfvctksw5Hz0poyd9T8OdcSqHDPcznPz9kE7+DTHlWMggg4yuzcgacdxRxqr6/kYcS2vLAUfbh8Pf75jsaf4/Fb/gaX4oOnXPU1NTkxITE9XQ0CBJevzxx5WQkKDJkydr5cqVCkP/BQAAAMAE7Za3i0GCNk+1tbVKTU1VfX29JOmjjz7S+vXr9fTTT2vr1q16++239frrr3tdJwAAAACEVdDmqaysTD6fTzExMZKkYcOG6fnnn9eAAQN09OhRNTU1adCgQZ4XCgAAAMBAjDz9n0AgoMsuu+y0dX379lVZWZmuu+46RUdHa8yYMZ4VCAAAAAAmCPk5T6fccsst2rVrl8455xwVFxe7WRMAAACAbsKyLE8Xk4TcPDU2NqqmpkaSFBkZqYSEBO3bt8/1wgAAAADAJCE3T3//+9+1ePFiHT16VJZl6cUXX9Sll17qRW0AAAAATNeL7nmKDHWD0aNHKz09XSkpKerTp48uu+wyzZo1y4vaAAAAAMAYnW6eKisrO16npKQoJSXFk4IAAAAAdCOGjQ55yfaEEQAAAADQm0RYYZjCwu/3d/UuAQAAAOP5fL5wlxCyT2Zd52n+19e97Gl+KMLSPEVGDXG0/bLcdEnSikCJERlbf2P/F5q04DrX6nAj49yH62xnHJo7SpJU/fCbtjOunHuFJHPOBxlkfFUOGe5nPPLLzY4yZmbeaMyxmJLxy58/5igjc9FtxhwLGWR8Vcbisw45ylh95FwjjqW15YCj7cOhNzVPnbpsr6mpSYmJiWpoaDht/caNGzV9+nRPCgMAAADQDfSi2faCNk+1tbVKTU1VfX39aev/8pe/qKTEWWcNAAAAAN1F0OaprKxMPp9PMTExHetaWlp077336s477/S0OAAAAACGa/d4MUjQqcoDgcDn1v385z/XtGnTNHToUE+KAgAAAADThDxV+euvv67GxkZNmzbNi3oAAAAAdCNWu+XpYpJOPyT3lG3btqmurk5TpkzRsWPH9PHHH+vuu+/WAw884EV9AAAAAGCEkJungoKCjte7du1ScXExjRMAAADQWxk2OuSlkC/bAwAAAIDeqNMjT5WVlZ9bN378eI0fP97VggAAAAB0I4bNiOelCMuyunycze/3d/UuAQAAAOP5fL5wlxCy/775Gk/z//npf/c0PxRhaZ4io4Y42n5ZbrokaUXA/kN63cx4qWSn7Yz49DjX6jDlWD5Yt8d2xvmzLpZkzvnoSRmrV663nbF4SZprdfSEjFM5ZLif4eR7Kp38rppyLKZkcE7J6A4ZBQWljjKys+do8dcPOspY/ck3jTgfrS0HHG0fDr2peQp5wggAAAAA6NCLLtvr1IQRTU1NSkxMVENDgyQpOztb8fHxmjJliqZMmaIdO3Z4WiQAAAAAhFvQkafa2lrl5eWpvr6+Y927776rjRs3KiYmxsvaAAAAABjOtAfZeinoyFNZWZl8Pl9Ho3T8+HEdPHhQOTk5SkpK0oMPPqj29l40VgcAAACgVwraPAUCAV122WUd7z/++GNdccUVys/PV1lZmd566y0988wznhYJAAAAwFDtHi8GCfkhucOGDdOvfvUrxcTEqH///po+fbp27rQ/QxsAAAAAdAchN0/79u3Tiy++2PHesixFRjJpHwAAANAbWe3eLiYJuXmyLEv5+fn65JNP9Omnn+qpp57SD3/4Qy9qAwAAAABjhDxkNGbMGKWnpys1NVWtra2Kj49XYmKiF7UBAAAAMJ1ho0Ne6nTzVFlZ2fH6tttu02233eZJQQAAAABgogjLsrp8Yna/39/VuwQAAACM5/P5wl1CyD6eFOdp/jkV5kxOF5bmKTJqiKPtl+WmS5JWBEqMyKh++E3bGVfOvcK1OtzISCXmVeQAACAASURBVHr4j7Yzts69RJK0q3SX7Yzxc8ZLMud8kEHGV+WQ4X7G6pXrHWUsXpJmzLH0pIxA/lpHGbk584w5FjJOz+hJv9ufRvzZUcYaa7QRx9LacsDR9uHQm5qnTk0Y0dTUpMTERDU0NEiS3n77bd1yyy1KSEjQwoUL1dLS4mmRAAAAAAzFc57+T21trVJTU1VfXy/pZCOVmZmp5cuX6/nnn5ckHpILAAAAoMcL2jyVlZXJ5/MpJiZGkvT6669r3LhxGjNmjCQpLy+PqcoBAACAXqo3Pecp6Gx7gUDgtPf79+/XgAEDdM899+iDDz7QJZdcoqVLl3pWIAAAAACYIOSH5La1tamqqkoLFy7Upk2bdPz4cZWUOLsxDgAAAED31JtGnkJuns455xyNHTtWw4YNU58+fTRp0iTt2bPHi9oAAAAAwBghN09XX3219u7dq8bGRknSq6++qgsvvND1wgAAAACYrzeNPAW95+mzzjvvPC1fvlwLFixQc3OzLrjgAi1ZssSL2gAAAACYzooIdwVdptPNU2VlZcfra665Rtdcc40X9QAAAACAkUIeeQIAAACAU0y7tM5LEZZlWV29U7/f39W7BAAAAIzn8/nCXULI/mvCNZ7mD/79v3uaH4qwNE+RUUMcbb8sN12StCJgf4p0NzM2Fm+xnfHjO25wrQ43MqofftN2xpVzr5Akvbr2NdsZ1877viRzzgcZZHxVDhnuZwTy1zrKyM2ZZ8yxmJLhxjnl90JGd8gY85s/O8r404LRRhxLa8sBR9uHQ+PV13qaf17Vq57mh6JTs+01NTUpMTFRDQ0N2rlzp6ZMmdKxXHHFFZo/f77XdQIAAADAV6qsrNTUqVM1adIk3XfffZKk6upqJSUlKT4+XkVFRY7yg97zVFtbq7y8PNXX10uS4uLiFBcXJ0k6fPiwUlNTlZ2d7agIAAAAAN2TKfc8ffTRR/L5fHr66ad19tlna+bMmdq5c6d8Pp82bNig8847T/Pnz9fOnTs7+plQBR15Kisrk8/nU0xMzOc+W7VqlVJSUjRixAhbOwcAAAAAN+zYsUOTJ0/W4MGD1bdvXxUVFal///4aPny4hg0bpsjISCUlJWn79u229xF05CkQCHzh+vr6eu3evftLPwcAAADQ81keP+fp6NGjOnr06OfWDxo0SIMGDep4v3//fvXt21cLFixQY2OjrrnmGo0aNUrR0dEdPxMTE6NDhw7ZrsX2VOVPPfWUbr31VkVFRdneOQAAAAB8lUceeUTFxcWfW3/HHXcoMzOz431bW5veeustbdiwQQMGDNDtt9+uM888UxER/9fcWZZ12vtQ2W6eXnnlFZWWltreMQAAAIDuz+t7nmbOnKnk5OTPrf/HUSdJOueccxQbG6uzzjpLknTddddp+/bt6tOnT8fPHD58+AtvR+qsTs2291lHjhzRiRMnNGzYMNs7BgAAAIBgBg0apKFDh35u+WzzdO2116qqqkpHjx5VW1ubXnvtNU2cOFEffvih9u/fr7a2Nm3btk0TJkywXYutkaeGhgYNHjzY9k4BAAAA9AxWu7f3PHXW2LFjNXfuXN1666369NNPddVVVyk1NVXnn3++MjMz1dzcrLi4OE2cONH2PjrdPFVWVna8vvjii1VWVmZ7pwAAAADgtptuukk33XTTaetiY2O1ZcsWV/IjLMuyXEkKgd/v7+pdAgAAAMbz+XzhLiFk/3nZv3ma/623XvE0PxRhaZ4io4Y42n5ZbrokaUWgxIiMkgfsj8Kl332La3W4kbF/3Tu2M4bPukiSVPHbV21nTJp/rSRzzkdPygjkr7WdkZszz7U6ekLGqRwy3M+4f/WjjjIWLp5hzLGYkuHk37508t+/KcdCBhlflZG+498dZZT88BojjqW15YCj7cOhNzVPtmfbAwAAAABT7nnqCp2aba+pqUmJiYlqaGiQJFVVVemGG25QYmKisrKy1NLS4mmRAAAAABBuQZun2tpapaamqr6+vmNdbm6uioqKtG3bNp04cULPPfeclzUCAAAAMJTVHuHpYpKgzVNZWZl8Pt9pD5Nqa2tTU1OT2tra1NzcrH79+nlaJAAAAACEW9B7ngKBwOfW/exnP9P06dM1cOBADR061NFc6QAAAAC6r66ffi58OnXP0z86fPiw1qxZo23btqmqqkpjx45VQUGBF7UBAAAAgDFCbp7eeustjR49Wt/61rd0xhln6JZbbtHu3bu9qA0AAACA4bjn6SuMHj1ae/bs0ccffyxJeuWVV3TRRRe5XhgAAAAA81lWhKeLSUJ+ztPIkSN11113acaMGerTp4+GDx+u5cuXe1EbAAAAABij081TZWVlx+vk5GQlJyd7UhAAAACA7sNqD3cFXSfCsrp+fgy/39/VuwQAAACM5/P5wl1CyP7y/13vaf533nvR0/xQhKV5iowa4mj7ZbnpkqQVgRIjMjb/+iXbGTfeHu9aHW5kHFq313bGubMulCRV/PZV2xmT5l8ryZzzYUpGIH+t7YzcnHmu1UHG6TmmZBQUlDrKyM6eY8yx3L/6UUcZCxfPMOZYTMlw4/vRk75jZPTcjJyM6Y4y8n+1wZU6nPw3W5JamhscbR8Of77A28cWjX5/u6f5oejUhBFNTU1KTExUQ8PJX+amTZs0efJkJSUl6b777lNra6unRQIAAABAuAVtnmpra5Wamqr6+npJ0gcffKAHHnhA69ev19atW9Xa2qoNGzZ4XScAAAAAA/Wm2faCNk9lZWXy+XyKiYmRJO3bt0/jxo3reH/ttdfq5Zdf9rZKAAAAAAizoM1TIBDQZZdd1vF+zJgxqq2tVWNjo9ra2rR9+/aOZz4BAAAA6F1600NyQ37O07e//W0tWrRIt99+u84880xNnDhR77zzjhe1AQAAAIAxQm6empubdfHFF2vz5s2SpIqKCg0bNsz1wgAAAACYr+vn7g6fTs2294+OHTumtLQ0NTU1qaWlRRs3btTkyZO9qA0AAAAAjBHyyNM///M/KyMjQz/60Y/U2tqqxMREJSUleVEbAAAAAMOZdl+SlzrdPFVWVna8vvnmm3XzzTd7UhAAAAAAmCjCsrr+KkW/39/VuwQAAACM5/P5wl1CyN49P9HT/O9+sM3T/FCEfM8TAAAAAPRGYRl5iowa4mj7ZbnpkqQVgRIjMn6/9nXbGRPmXeVaHW5kfK20znbG/84ZJUna97v/sJ3xL7PHSTLnfJiSEchfazsjN2eeaxmmnI9wZ5zKIcP9jIKCUkcZ2dlzjDkWUzLcOKf8XsjoDhmLzzrkKGP1kXONOJbWlgOOtg+Hd77t7fwHF3241dP8UHRq5Km4uFgJCQlKSEjQqlWrJEnV1dVKSkpSfHy8ioqKPC0SAAAAAMItaPNUXV2tqqoqlZeXa/Pmzdq7d6+2bdumnJwcPfTQQ3rhhRf07rvvaufOnV1RLwAAAACDWJa3i0mCNk/R0dFaunSpoqKi1LdvX40cOVL19fUaPny4hg0bpsjISCUlJWn79u1dUS8AAAAAhEXQ5mnUqFEaN+7kfSj19fWqqKhQRESEoqOjO34mJiZGhw45u84UAAAAQPfTbkV4upik0895qqur0/z585WVlaU+ffqovr6+4zPLshQRYdaBAQAAAPCeZViD46VOTRhRU1OjtLQ0LVq0SMnJyRo8eLAOHz7c8fnhw4cVExPjWZEAAAAAEG5Bm6fGxkZlZGRozZo1SkhIkCSNHTtWH374ofbv36+2tjZt27ZNEyZM8LxYAAAAAGbpTRNGBL1sr7S0VM3NzSosLOxYl5KSosLCQmVmZqq5uVlxcXGaOHGip4UCAAAAQDgFbZ7y8vKUl5f3hZ9t2bLF9YIAAAAAdB+mTergpQjL6vrBML/f39W7BAAAAIzn8/nCXULI3hp6o6f5lzVs9jQ/FGFpniKjhjjaflluuiRpRaDEiIxdpbtsZ4yfM961OtzIOLRur+2Mc2ddKEn6/drXbWdMmHeVJHPOhykZgfy1tjNyc+a5lmHK+Qh3xqkcMtzPWL1yvaOMxUvSjDkWUzJ++fPHHGVkLrrN0d8P6eTfEFPOBxk9N2PJlEsdZax8rsaIY2ltOeBo+3D4w5BkT/MvP1DuaX4oOjXbHgAAAAD0dp1qnoqLi5WQkKCEhAStWrWqY/2nn36qmTNnatcu+yMvAAAAALqv3vSQ3KDNU3V1taqqqlReXq7Nmzdr79692rFjhz744ANNnz5db7/9dlfUCQAAAABhFbR5io6O1tKlSxUVFaW+fftq5MiROnjwoJ555hnNnTtXY8eO7Yo6AQAAABjI8ngxSdCpykeNGtXxur6+XhUVFXriiSc0YsQISdIjjzziWXEAAAAAYIqgzdMpdXV1mj9/vrKysjoaJwAAAAC9m2n3JXmpUxNG1NTUKC0tTYsWLVJysrdTEQIAAACAiYKOPDU2NiojI0NFRUWKjY3tipoAAAAAdBNWLxp5Cto8lZaWqrm5WYWFhR3rUlJSlJqa6mlhAAAAAGCSoM1TXl6e8vLyvvTzDRs2uFoQAAAAgO6jPdwFdKEIy7K6fAZAv9/f1bsEAAAAjOfz+cJdQsheG3yTp/nf/69nPM0PRViap8ioIY62X5abLklaESgxImPzr1+ynXHj7fGu1eFGxpF179vOOGvWBa5lmHI+yCDjq3LIIKM7ZBQUlDrKyM6eo0D+WkcZuTnzjDkfZPTcjKyrznWUser1Q0YcS2vLAUfbh8PvB9/saf6E/3ra0/xQdGqq8uLiYlVUVEiS4uLilJWVpaeeekobNmxQRESEvvvd78rv9ysqKsrTYgEAAAAgXIJOVV5dXa2qqiqVl5dr8+bN2rt3r0pKSlRaWqonn3xSW7ZsUXt7ux5//PGuqBcAAACAQdotbxeTBB15io6O1tKlSztGlUaOHKmWlhb5fD4NHDhQkjR69GgdPHjQ20oBAAAAGKddTFXeYdSoUR2v6+vrVVFRoSeeeEIjRoyQJB05ckSPPfaYCgoKPCsSAAAAAMKtU/c8SVJdXZ3mz5+vrKysjsbp0KFDmjt3rqZNm6bx48d7VSMAAAAAQ1m9aOQp6D1PklRTU6O0tDQtWrRIycnJkqS//vWvSklJUXJysjIyMjwtEgAAAADCLejIU2NjozIyMlRUVKTY2FhJUlNTk+bMmaO7775bN954o+dFAgAAADBTb3pIbtDmqbS0VM3NzSosLOxYN3nyZH388cdat26d1q1bJ0n6wQ9+oLvuusu7SgEAAAAgjII2T3l5ecrLy/vc+vnz53tSEAAAAIDuozfd8xRhWVaXz57u9/u7epcAAACA8Xw+X7hLCNlL56Z4mh9/6ElP80PR6dn2AAAAAOCzetM9T2EZeYqMGuJo+2W56ZKkFYESIzJeKtlpOyM+Pc61OtzIOLLufdsZZ826QJJU/fCbtjOunHuFJHPOhykZgfy1tjNyc+a5lmHK+Qh3xqkcMtzPKCgodZSRnT3HmGMxJWP1yvWOMhYvSTPmWNzIcPK3UDr599CUYyHj9IyfRvzZUcYaa7QRx9LacsDR9uGw3eORp4ndbeSpuLhYFRUVkqS4uDhlZWXp8ccf12OPPSbLsjrWRUT0nusdAQAAAPSukaegzVN1dbWqqqpUXl6uiIgIzZ07V+vXr9fjjz+uzZs3q1+/frrtttv0+uuv6+qrr+6KmgEAAACgywVtnqKjo7V06VJFRUVJkkaOHKmIiAg9//zz6tu3r/77v/9bTU1NGjRokOfFAgAAADBLb5pt74xgPzBq1CiNGzdOklRfX6+KigrFxcWpb9++Kisr03XXXafo6GiNGTPG82IBAAAAIFyCNk+n1NXVafbs2crKytKIESMkSbfccot27dqlc845R8XFxV7VCAAAAMBQ7RHeLibpVPNUU1OjtLQ0LVq0SMnJyWpsbFRNTY0kKTIyUgkJCdq3b5+nhQIAAABAOAVtnhobG5WRkaE1a9YoISFBkvT3v/9dixcv1tGjR2VZll588UVdeumlnhcLAAAAwCztivB0MUnQCSNKS0vV3NyswsLCjnUpKSlKT09XSkqK+vTpo8suu0yzZs3ytFAAAAAACKegzVNeXp7y8vK+8LOUFG8fiAUAAADAbFa4C+hCEZZldfnx+v3+rt4lAAAAYDyfzxfuEkK2efCtnubf+F+Pe5ofirA0T5FRQxxtvyw3XZK0IlBiRMahdXttZ5w760LX6nAjo/Z3NbYzxs4+ed+b9bs/286ImD1akjnnw5SMQP5a2xm5OfMkSfevftR2xsLFMySZcz7CnXEqhwz3MwoKSh1lZGfPMeZYTMl46P4nHGX8ZGGqo78f0sm/IaacDzJ6bsYUB/8PI0nPzb7UiGNpbTngaPtw2ORx8zTVoOYp6GV7AAAAAPBl2iPMmtTBS52aqry4uFgJCQlKSEjQqlWrTvts48aNmj59uifFAQAAAIApgjZP1dXVqqqqUnl5uTZv3qy9e/dqx44dkqS//OUvKilxNjQJAAAAoPuyPF5MErR5io6O1tKlSxUVFaW+fftq5MiROnjwoFpaWnTvvffqzjvv7Io6AQAAACCsgt7zNGrUqI7X9fX1qqio0BNPPKGf//znmjZtmoYOHeppgQAAAADM1R7uArpQp+55kqS6ujrNnj1bWVlZOnDggBobGzVt2jQvawMAAAAAY3SqeaqpqVFaWpoWLVqk5ORkbdu2TXV1dZoyZYry8vL07rvv6u677/a6VgAAAACGaY/wdjFJ0Mv2GhsblZGRoaKiIsXGxkqSCgoKOj7ftWuXiouL9cADD3hXJQAAAACEWdDmqbS0VM3NzSosLOxYl5KSotTUVE8LAwAAAGC+dhk2POShoM1TXl6e8vLyvvTz8ePHa/z48a4WBQAAAACmibAsq8unT/f7/V29SwAAAMB4Pp8v3CWEbOM3f+xp/o8PbvQ0PxRhaZ4io4Y42n5ZbrokaUXA/gN6T2UE8tfazsjNmSdJ2r/uHdsZw2ddJMmdY3Ejw41j2VW6y3bG+DknRzFNOR89KcON77opxxLujFM5ZLifcf/qRx1lLFw8w5hjMSXjlz9/zFFG5qLbHP39kE7+DTHlfJDRczMWndjjKOPnZ15sxLG0thxwtH049KbmKehle5JUXFysiooKSVJcXJyysrKUnZ2tmpoa9e/fX5J0xx136Ic//KF3lQIAAAAwjmkz4nkpaPNUXV2tqqoqlZeXKyIiQnPnztWOHTv07rvvauPGjYqJiemKOgEAAAAgrII+5yk6OlpLly5VVFSU+vbtq5EjR+rgwYM6ePCgcnJylJSUpAcffFDt7b3p2cIAAAAAJKnd48WOlStXaunSpZJODgYlJSUpPj5eRUVFNhNPCto8jRo1SuPGjZMk1dfXq6KiQt///vd1xRVXKD8/X2VlZXrrrbf0zDPPOCoEAAAAAJx64403VF5eLkk6ceKEcnJy9NBDD+mFF17Qu+++q507d9rODto8nVJXV6fZs2crKytL559/vn71q18pJiZG/fv31/Tp0x0VAQAAAKB7sjxeQvE///M/Kioq0oIFCyRJe/bs0fDhwzVs2DBFRkYqKSlJ27dvt32snZowoqamRnfeeadycnKUkJCgffv2qb6+Xtdff70kybIsRUZ2KgoAAAAAOu3o0aM6evTo59YPGjRIgwYNOm3dvffeq3vuuUeNjY2SpL/97W+Kjo7u+DwmJkaHDh2yXUvQjqexsVEZGRkqKipSbGyspJPNUn5+vq644goNGDBATz31lJKTk20XAQAAAKB78nq2vUceeUTFxcWfW3/HHXcoMzOz4/3TTz+t8847T7Gxsdq0adPJ2trbFRHxfwValnXa+1AFbZ5KS0vV3NyswsLCjnUpKSlKT09XamqqWltbFR8fr8TERNtFAAAAAMAXmTlz5hcO1Hx21OmFF17Q4cOHNWXKFH3yySc6duyYDhw4oD59+nT8zOHDhx3NFh60ecrLy1NeXt4XfnbbbbfZ3jEAAACA7s/rObe/6PK8L7Ju3bqO15s2bdLu3bvl9/sVHx+v/fv3a+jQodq2bZumTZtmu5YIy7JCvQ/LMb/f39W7BAAAAIzn8/nCXULI1g79saf58xo2hrzNqeapsLBQb7zxhgoKCtTc3Ky4uDhlZ2fbvnQvLM1TZNQQR9svy02XJK0IlBiR8cfSP9jOuGTO5a7V4UbGGb/7s+2M9tmjJUm7SnfZzhg/Z7wkc85HT8pYvXK97YzFS9Jcq6MnZJzKIcP9DCffU+nkd9WUYzElo6Cg1FFGdvYcY46FjJ6bEchf6ygjN2eessY4itCqP5nx34bWlgOOtg+H33rcPM230Tx5hSnyAAAAANhmeTxhhEk69Zyn4uJiJSQkKCEhQatWrZIkvf3227rllluUkJCghQsXqqWlxdNCAQAAACCcgjZP1dXVqqqqUnl5uTZv3qy9e/eqvLxcmZmZWr58uZ5//nlJ0jPPPON5sQAAAADM0u7xYpKgl+1FR0dr6dKlioqKkiSNHDlSBw4c0Lhx4zRmzMmLS/Py8tTW1uZtpQAAAAAQRkFHnkaNGqVx48ZJkurr61VRUaGoqCgNGDBA99xzj6ZMmaJf/vKXnZo+EAAAAEDP0ptGnjp1z5Mk1dXVafbs2crKylJbW5uqqqq0cOFCbdq0ScePH1dJibOZRQAAAADAZJ1qnmpqapSWlqZFixYpOTlZ55xzjsaOHathw4apT58+mjRpkvbs2eN1rQAAAAAMY3m8mCRo89TY2KiMjAytWbNGCQkJkqSrr75ae/fuVWNjoyTp1Vdf1YUXXuhtpQAAAAAQRkEnjCgtLVVzc7MKCws71qWkpGj58uVasGCBmpubdcEFF2jJkiWeFgoAAADAPO296DlPQZunvLw85eXlfeFn11xzjdv1AAAAAICRIizL6vJLCf1+f1fvEgAAADCez+cLdwkhK/rWjz3Nv+c/N3qaH4qwNE+RUUMcbb8sN12StCJgf4Y/NzNeXfua7Yxr533ftTrcyPhaaZ3tjP+dM0qS9MfSP9jOuGTO5ZLMOR9kkPFVOWSQ0R0yHrr/CUcZP1mYqkD+WkcZuTnzjDkfZPTcjKwhnzjKWHXg60YcS2vLAUfbh0Nvap6CXrYHAAAAAF/GtGcxealTzVNxcbEqKiokSXFxcRo/frzuv//+js8PHTqksWPH6re//a03VQIAAABAmAVtnqqrq1VVVaXy8nJFRERo7ty5+t73vqfnnntOknT48GGlpqYqOzvb82IBAAAAmMW0ZzF5KWjzFB0draVLlyoqKkqSNHLkSB08eLDj81WrViklJUUjRozwrEgAAAAACLegzdOoUaM6XtfX16uiokJPPPFEx/vdu3crEAh4VyEAAAAAY/Wm5zyd0dkfrKur0+zZs5WVldUxyvTUU0/p1ltv7RiVAgAAAICeqlPNU01NjdLS0rRo0SIlJyd3rH/llVc0efJkz4oDAAAAYLZ2jxeTBL1sr7GxURkZGSoqKlJsbGzH+iNHjujEiRMaNmyYpwUCAAAAgAmCNk+lpaVqbm5WYWFhx7qUlBRdeOGFGjx4sKfFAQAAADAbs+39g7y8POXl5X3hZ2VlZa4XBAAAAKD7aO9F7VOEZVldfrR+v7+rdwkAAAAYz+fzhbuEkAWG3+Zpfu7+xzzND0VYmqfIqCGOtl+Wmy5JWhEoMSLjk3Xv2874+qwLXKvDjYzNv37JdsaNt8dLks5+uM52xv+be3JqfFPOR0/KKCgotZ2RnT3HtTp6QsapHDJOzwjkr3WUkZszz9H3VDr5XTXlfJiSsfU3LzvKSFpwne5f/aijjIWLZxhzPtz4nppyLD3pfLhRx8Tf1jrK2D5/rBG/l9aWA462D4cVHjdPywxqnjo1215xcbESEhKUkJCgVatWSZKqqqp0ww03KDExUVlZWWppafG0UAAAAAAIp6DNU3V1taqqqlReXq7Nmzdr79692rFjh3Jzc1VUVKRt27bpxIkTeu6557qiXgAAAAAGsTxeTBJ0wojo6GgtXbq040G4I0eO1MGDB9XW1qampia1tbWpublZ/fr187xYAAAAAAiXoM3TqFGjOl7X19eroqJCTzzxhIYMGaLp06dr4MCBGjp0qCZOnOhpoQAAAADMY9qDbL3UqXueJKmurk6zZ89WVlaWvva1r2nNmjXatm2bqqqqNHbsWBUUFHhZJwAAAACEVaeap5qaGqWlpWnRokVKTk7WW2+9pdGjR+tb3/qWzjjjDN1yyy3avXu317UCAAAAMEx7hLeLSYI2T42NjcrIyNCaNWuUkJAgSRo9erT27Nmjjz/+WJL0yiuv6KKLLvK2UgAAAAAIo6D3PJWWlqq5uVmFhYUd61JSUnTXXXdpxowZ6tOnj4YPH67ly5d7WigAAAAA87QbNyeed4I2T3l5ecrLy/vCz5KTk10vCAAAAABMFGFZVpe3in6/v6t3CQAAABjP5/OFu4SQ5Y641dP8QP3jnuaHIizNU2TUEEfbL8tNlyStCJQYkfHH0j/YzrhkzuWu1eFGxpO/et52RkrGyXvietL56EkZ969+1HbGwsUzXKujJ2ScyiHD/YyCglJHGdnZc4w5FlMyVq9c7yhj8ZI0R38/pJN/Q0w5H2SYmRHIX+soIzdnnn78dLWjjI03X2nE+WhtOeBo+3DoTc1T0Mv2AAAAAODL9KbnPHWqeSouLlZFRYUkKS4uTllZWdq0aZMefvhh9enTR+PHj9fSpUsVGUkvBgAAAKBnCjpVeXV1taqqqlReXq7Nmzdr7969Kikp0QMPPKD169dr69atam1t1YYNG7qiXgAAAAAGaZfl6WKSoM1TdHS0li5dqqioKPXt21cjR45US0uLxo0bp5iYGEnStddeq5dfftnzYgEAAAAgXII2T6NGjdK4ceMkSfX19aqoqNDkyZNVW1urxsZGtbW1afv27R0PzAUAAADQe1geLybp9E1KdXV1mj9/vrKyqEmSkwAAIABJREFUsnT++edr0aJFuv3223XmmWdq4sSJeuedd7ysEwAAAADCqlPNU01Nje68807l5OQoISFBzc3Nuvjii7V582ZJUkVFhYYNG+ZpoQAAAADM05tm2wt62V5jY6MyMjK0Zs0aJSScfI7PsWPHlJaWpqamJrW0tGjjxo2aPHmy58UCAAAAMEtvmjAi6MhTaWmpmpubVVhY2LEuJSVFGRkZ+tGPfqTW1lYlJiYqKSnJ00IBAAAAIJyCNk95eXnKy8v7ws9uvvlm1wsCAAAA0H2YNTbkrQjLsrr8eP1+f1fvEgAAADCez+cLdwkhu2dEiqf5RfVPepofirA0T5FRQxxtvyw3XZK0IlBiREbLun22M6Jm/YtrdbiRcWTd+7Yzzpp1gSTp7IfrbGf8v7mjJJlzPsgg46tyyHA/4/7VjzrKWLh4hjHHYkpGQUGpo4zs7DmuZPSk82HKsZBxekbqk284yngiJdaIY2ltOeBo+3C4y+Pm6RcGNU9BJ4yQpF/84heaPHmyEhIStG7dOklSdXW1kpKSFB8fr6KiIk+LBAAAAIBwC3rP0+7du/Xmm29qy5Ytam1t1eTJkxUbG6v/v727D4/pzvsH/k5nEk3WutVWwpLq3W4tPy12tWG2mGoZzORJpAT1rGhrovSqh0hlo0TpdtME7ZbVuBPtRRAV6qFt2toikbJdc2uzVmOjG2YjYgnFnMzk+/vDJTfpxMw5Mydz8H655ro6E+c9n3N8cs58e+Z8T0pKCvLy8tC+fXtMmzYNe/fuhdFobI6aiYiIiIhII8RddNWTxzNPUVFRyM3NhV6vR01NDVwuF2pra9GpUydERkZCr9cjJiYGu3fvbo56iYiIiIiIAsKrr+0FBwcjOzsbFosFBoMBZ86cQdu2bRt+Hh4ejqqqKtWKJCIiIiIibapX+aElXg2eACA5ORnFxcWw2+2oqKhAUFBQw8+EEDc9JyIiIiIiutN4vOapvLwckiSha9euCA0Nhclkwu7du6HT6Rr+TnV1NcLDw1UtlIiIiIiItKee1zz9n8rKSqSmpkKSJEiShKKiIiQlJeGf//wnTp48CZfLhR07dqB///7NUS8REREREVFAeDzzZDQaYbPZEB8fD51OB5PJBIvFgjZt2sBqtcLhcMBoNGLIkCHNUS8REREREWnI3XPeyYvBEwBYrVZYrdabXjMYDCgsLFSlKCIiIiIiIq3xavBERERERETkzt10zVOQEKLZ1zY9Pb2535KIiIiISPPS0tICXYJs0x58VtX89yo2qZovR0AGT/qQDj4t/9qCqQCA15es1kTG9j99pjgjZvpAv9Xhj4yTOf+rOKPTxMcAAFU53yrOiJjYDYB2tgczmHGrHGb4P2Pp0rU+ZcyfP1kz66KVjCUZa3zKWJDyvF8ytLI9mHHnZsyuPuhTxh/b9tbEujilUz4tHwjPqzx4WqOhwZNX93nKysqC2WyGxWJBTk5Ow+t1dXUYP348Dh70rVmJiIiIiIi0zuM1T6WlpSgpKUFhYSGcTifMZjOMRiMAICUlBd99953qRRIRERERkTaJu+iaJ49nnqKiopCbmwu9Xo+amhq4XC6EhYVh8+bNmDJlCnr06NEcdRIREREREQWUV1/bCw4ORnZ2NiwWCwwGAyIiIjBnzhwMHDhQ7fqIiIiIiEjD6lV+aIlXgycASE5ORnFxMex2O/Lz89WsiYiIiIiIbhNC5T9a4nHwVF5ejrKyMgBAaGgoTCYTjh07pnphREREREREWuJx8FRZWYnU1FRIkgRJklBUVIRevXo1R21ERERERKRxd9PX9jzOtmc0GmGz2RAfHw+dTgeTyQSLxdIctREREREREWmGx8ETAFitVlitVrc/y8vL82tBRERERER0+6gX2rouSU1BQjT/2qanpzf3WxIRERERaV5aWlqgS5BtbKcEVfPzThaomi9HQAZP+pAOPi3/2oKpAIDXl6zWRMYXa75SnDHg+X5+q8MfGedyyhRntJnYFQDwlzX7FWf0f/5JANrZHlrJWJKxRnHGgpTn/Zahle0R6IzrOczwf8Yf38z1KWP2q+M0sy7MYAYzmjfD+tfPfcpY8dunNbEuTumUT8sHwnMqD57Wa2jw5PVU5URERERERHczr655ysrKwp49exAUFITExERMnDgRGzduRF5eHoKCgvDoo48iPT0dISEhatdLREREREQaUq+xezGpyeOZp9LSUpSUlKCwsBBbtmxBXl4eTpw4gbVr12LDhg0oLCxEfX09Pvzww+aol4iIiIiIKCA8nnmKiopCbm4u9Ho9qqqq4HK50KJFC6SlpaFly5YAgM6dO+P06dOqF0tERERERNoieObpZsHBwcjOzobFYoHBYMAvf/lLPPnktQv7z507hw8++ADPPPOMqoUSEREREREFktcTRiQnJ6O4uBh2ux35+fkAgKqqKowfPx7Dhw9H7969VSuSiIiIiIi0qV7lh5Z4HDyVl5ejrOza9NWhoaEwmUw4duwYysvLkZSUhGHDhuGll15SvVAiIiIiIqJA8jh4qqysRGpqKiRJgiRJKCoqQvfu3TF58mTMnDkTkyZNao46iYiIiIhIg+ohVH1oiccJI4xGI2w2G+Lj46HT6WAymXD+/HmcPXsWOTk5yMnJAQA8/fTTmDlzpuoFExERERERBYJX93myWq2wWq03vTZhwgQ16iEiIiIiotvI3TTbXpAQotnXNj09vbnfkoiIiIhI89LS0gJdgmyJnWJVzd98slDVfDkCMnjSh3TwafnXFkwFALy+ZLUmMtZmbVacMXlmot/q8EfG+pXKm/O5Gdd+cVa89YHiDOsrYwBoZ3swgxm3ymEGM5jBDGZoJ+Ojdz/xKSP+BZNf6liSscanDMlR6dPygZCg8uCpQEODJ6+mKs/KyoLZbIbFYmm4xunDDz+ExWKB2WzGsmXLEIAxGBERERERUbPxeM1TaWkpSkpKUFhYCKfTCbPZDKPRiHXr1uGjjz5CixYtMGbMGOzfvx99+/ZtjpqJiIiIiEgj7qaTKB7PPEVFRSE3Nxd6vR41NTVwuVwICwvDxx9/jLCwMNTW1uLSpUto1apVc9RLREREREQUEF59bS84OBjZ2dmwWCwwGAyIiIhAcHAw8vPzMXDgQLRt2xZdunRRu1YiIiIiItKYu+k+T14NngAgOTkZxcXFsNvtyM/PBwCMGDECBw8exP3334+VK1eqViQREREREWlTvcoPOVauXAmLxQKLxYLly5cDAA4cOICYmBiYTCZkZmb6sKZeDJ7Ky8tRVlYGAAgNDYXJZMKRI0dw+PBhAIBer4fFYsGxY8d8KoSIiIiIiEipAwcOYN++fdi6dSs++ugjfPvtt9ixYwdSUlLwzjvvYOfOnTh69Cj27t2r+D08Dp4qKyuRmpoKSZIgSRKKiorQsWNHvPrqq6itrYUQAnv27EGvXr0UF0FERERERLcnofIfb7Vt2xbz5s1DSEgIgoOD8fDDD6OiogKdOnVCZGQk9Ho9YmJisHv3bsXr6nG2PaPRCJvNhvj4eOh0OphMJrz44oto06YNkpKSoNPp8Pjjj2PixImKiyAiIiIiInKntrYWtbW1P3m9VatWN01a98gjjzT8d0VFBXbt2oXnnnsObdu2bXg9PDwcVVVVimvxOHgCAKvVCqvVetNrSUlJSEpKUvzGRERERER0+1N7Uof/+Z//cTu/wowZM34yRgGA48ePY9q0aZgzZw50Oh0qKioafiaEQFBQkOJavBo8ERERERERBcL48eMxbNiwn7zu7lZJhw8fRnJyMlJSUmCxWFBaWorq6uqGn1dXVyM8PFxxLUEiAHe1Sk9Pb+63JCIiIiLSvLS0tECXINvQyKGq5u/61y6v/p7dbsewYcOQmZkJg8EAAHA4HDCZTMjNzUXHjh0xbdo0DB8+HEOHKqs5IIMnfUgHn5Z/bcFUAMDrS1ZrIuOjdz9RnBH/gslvdfgjI+79w4oztk26NmnIJ6uVz2BimmoEoJ3tcSdlvLlsneKMV+dO8Fsdd0LG9Rxm+D9jxVsf+JRhfWWMZtblTspYkrHGp4wFKc9rZl2YcedmzPphn08ZmQ/01cS6OKVTPi0fCFoZPC1evBhbtmzBAw880PBaUlISHnzwQSxduhQOhwNGoxHz589X/NU9r762l5WVhT179iAoKAiJiYk3TQ6xfv167NmzB3l5eYoKICIiIiKi25fcezGpJTU1FampqW5/VlhY6Jf38Dh4Ki0tRUlJCQoLC+F0OmE2m2E0GvHQQw/h+++/x+rVq9GpUye/FENERERERKRVHu/zFBUVhdzcXOj1etTU1MDlciEsLAySJGHhwoVITk5ujjqJiIiIiEiDtHKfp+bgcfAEAMHBwcjOzobFYoHBYEBERATeeustDB8+HJGRkWrXSEREREREFHBeDZ4AIDk5GcXFxbDb7di4cSPsdjuGDx+uZm1ERERERKRx9RCqPrTE4zVP5eXlkCQJXbt2RWhoKEwmE44cOYLjx48jLi4Oly9fxtmzZ/Hyyy/j7bffbo6aiYiIiIiImp3HM0+VlZVITU2FJEmQJAlFRUXo27cvdu3ahW3btmHx4sV49NFHOXAiIiIiIroLCSFUfWiJxzNPRqMRNpsN8fHx0Ol0MJlMsFgszVEbERERERGRZnh1nyer1Qqr1er2Z71790bv3r39WhQREREREd0etHZdkpqCRADOhaWnpzf3WxIRERERaV5aWlqgS5BtQMdBquZ/UfmpqvlyBGTwpA/p4NPyry2YCgB4fclqTWQcef+w4owek3r5rQ5/ZDzxTpnijK9f7AoA2PTOLsUZz744FIB2tgczmHGrHGb4P+PNZet8ynh17gTNrItWMpYuXetTxvz5k7EkY41PGQtSntfM9mDGnZsx8+9f+pSR1eUpTayLUzrl0/KB8FTHgarmf1n5mar5cng9VTkREREREdHdzKtrnrKysrBnzx4EBQUhMTEREydOxPz583H48GGEhoYCAGbMmIFBg9Q9ZUdERERERNpSr7EZ8dTkcfBUWlqKkpISFBYWwul0wmw2w2g04ujRo1i/fj3Cw8Obo04iIiIiItKgu2fo5MXX9qKiopCbmwu9Xo+amhq4XC7ce++9OH36NFJSUhATE4Ps7GzU19c3R71EREREREQB4dU1T8HBwcjOzobFYoHBYIDT6USfPn2QkZGB/Px8HDp0CJs3b1a7ViIiIiIi0ph6CFUfWuL1hBHJyckoLi6G3W5HcXExVq1ahfDwcISGhmLs2LHYu3evmnUSEREREREFlMdrnsrLyyFJErp27YrQ0FCYTCbs3LkTrVu3xuDBgwEAQgjo9V7NPUFERERERHcQrZ0dUpPHM0+VlZVITU2FJEmQJAlFRUV44oknkJGRgQsXLqCurg4bN27kTHtERERERHRH83i6yGg0wmazIT4+HjqdDiaTCTNmzMB9992HUaNGwel0wmQyITo6ujnqJSIiIiIiDRGcqvxmVqsVVqv1ptfGjBmDMWPGqFIUERERERGR1gSJAAwV09PTm/stiYiIiIg0Ly0tLdAlyBb1S6Oq+aWntTMxXUAGT/qQDj4t/9qCqQCA15es1kTGpnd2Kc549sWhfqvDHxk/W3tcccaPkx8BABz4c4nijN9N6QNAO9vjTspYkrFGccaClOf9VsedkHE9hxn+z1ib5dttLybPTNTMumgl481l63zKeHXuBCxdutanjPnzJ2tme/iyLwSu7Q+1si7MuDnjlUt/8ynjrZY9NbEuTumUT8sHwt00ePJqqvKsrCyYzWZYLBbk5OQAAL755huMGDECFosFs2fPhiRJqhZKRERERETaI1T+oyUeB0+lpaUoKSlBYWEhtmzZgry8PPz973+H1WrFokWL8PHHHwMAb5JLRERERER3NI8TRkRFRSE3Nxd6vR5VVVVwuVwoKytDz5490aVLFwBAamoqXC6X6sUSEREREZG23E2z7Xn1tb3g4GBkZ2fDYrHAYDCguroaYWFhmDVrFuLi4rBixQq0atVK7VqJiIiIiIgCxqvBEwAkJyejuLgYdrsdkiRh3759mD17NgoKCnDlyhWsXu3bxXFERERERHT7qYdQ9aElHgdP5eXlKCsrAwCEhobCZDJh9erV6NGjByIjI6HT6TB06FDYbDbViyUiIiIiIgoUj4OnyspKpKamQpIkSJKEoqIiLFq0CN9++y3sdjsA4IsvvkC3bt1UL5aIiIiIiLRFCKHqQ0s8ThhhNBphs9kQHx8PnU4Hk8mE+Ph4tG7dGtOnT4fD4UDXrl0xd+7c5qiXiIiIiIgoIDwOngDAarXCarXe9NpTTz2Fp556So2aiIiIiIjoNqG165LU5PWEEURERERERHezIBGALxKmp6c391sSEREREWleWlpaoEuQrXs7g6r5tn8Xq5ovR0AGT/qQDj4t/9qCqQCA15conx7dnxlvLlunOOPVuRP8Voc/Mv6yZr/ijP7PPwkAWJu1WXHG5JmJALSzPe6kjCUZaxRnLEh53m913AkZ13OY4f8MX/anwLV9qlbWhRnMYEbzZrx84i8+Zbz9UH9NrItTOuXT8oHwaEQfVfOPVpWomi+HV9c8ZWVlYc+ePQgKCkJiYiIeeugh/PGPf2z4eVVVFXr06IH33ntPtUKJiIiIiIgCyePgqbS0FCUlJSgsLITT6YTZbMaf//xnbNu2DQBQXV2NUaNGYf78+aoXS0RERERE2iI4YcT/iYqKQm5uLvR6PWpqauByuRAWFtbw8+XLlyMpKQkPPvigmnUSEREREREFlFez7QUHByM7OxsWiwUGgwEREREAgIqKCpSWlmLcuHGqFklERERERNpUL4SqDy3xeqry5ORkFBcXw263Iz8/HwCwceNGjB49GiEhIaoVSEREREREpAUeB0/l5eUoKysDAISGhsJkMuHYsWMAgKKiIpjNZnUrJCIiIiIizRIq/9ESj4OnyspKpKamQpIkSJKEoqIi9OrVC+fOncPVq1cRGRnZHHUSEREREREFlMfZ9oxGI2w2G+Lj46HT6WAymWCxWGCz2dCuXbvmqJGIiIiIiDRKa9clqcmr+zxZrVZYrdabXuvevXvDtU9ERERERER3uiAhmn+omJ6e3txvSURERESkeWlpaYEuQbZH2vZSNf949WFV8+UIyOBJH9LBp+VfWzAVAPD6ktWayPjr2q8VZ/x28hN+q8MfGU+8U6Y44+sXuwIA1mZtVpwxeWYiAO1sD2Yw41Y5zLg5Y0nGGp8yFqQ8jzeXrfMp49W5EzSzPe6kjKVL1/qUMX/+ZM2sCzPu3IxXI875lPFmVRtNrItTOuXT8oFwNw2evPraHhERERERkTt30zVPXt3nKSsrC2azGRaLBTk5OQCAffv2ITY2FtHR0ZgzZw4kSVK1UCIiIiIiokDyOHgqLS1FSUkJCgsLsWXLFuTl5eHEiRNYsGABMjMzsWPHDly9ehXbtm1rjnqJiIiIiEhDeJ+nG0RFRSE3Nxd6vR41NTVwuVwICwuDy+XCpUuX4HK54HA40KJFi+aol4iIiIiIKCC8uuYpODgY2dnZeP/99zFkyBBERETg97//PcaOHYuWLVuiY8eOGDJkiNq1EhERERGRxghRH+gSmo1X1zwBQHJyMoqLi2G327Fq1Sr84Q9/wI4dO7Bv3z706NEDS5cuVbNOIiIiIiKigPI4eCovL0dZ2bXpq0NDQ2EymbBr1y507twZDzzwAO655x6MGDECpaWlqhdLRERERETaUg+h6kNLPA6eKisrkZqaCkmSIEkSioqKEBsbC5vNhrNnzwIAioqK8Nhjj6leLBERERERUaB4vObJaDTCZrMhPj4eOp0OJpMJ06ZNQ3h4OMaNGwedTodOnTph0aJFzVEvERERERFpiLiL7vPk1YQRVqsVVqv1pteGDRuGYcOGqVIUERERERHdHrT21To1BYkADBXT09Ob+y2JiIiIiDQvLS0t0CXI1rHNo6rmV547qmq+HAEZPOlDOvi0/GsLpgIAXl+yWhMZ373/jeKM/zfpN36rwx8Zv/jzccUZNVMeAQAcXHtQcUbvyb0B+GddlmSsUZyxIOV5v9WhlQxuD/9lXM9hhv8z/vhmrk8Zs18dp5l10UqGL7/7wLXff39kaGV7MOPOzXj5xF98ynj7of6aWBendMqn5QOhw33dVM0/9Z9vVc2Xw6upyrOysmA2m2GxWJCTkwMAKCgogNlsRkxMDBYvXgyn06lqoURERERERIHkcfBUWlqKkpISFBYWYsuWLcjLy8OJEyfw9ttvY926ddi+fTucTify8vKao14iIiIiItKQeiFUfWiJx8FTVFQUcnNzodfrUVNTA5fLBZvNhp49eyI8PBwAMGDAAHz22WeqF0tERERERBQoXn1tLzg4GNnZ2bBYLDAYDOjevTuOHDkCu90Ol8uF3bt3N9zziYiIiIiI7h5C5T9a4tXgCQCSk5NRXFwMu92Or7/+Gq+88gpeeOEFjBkzBr/+9a8RHBysZp1EREREREQB5XHwVF5ejrKyMgBAaGgoTCYTbDYbunfvjo8++ggbNmxAREQEIiMjVS+WiIiIiIi0RQih6kNLPA6eKisrkZqaCkmSIEkSioqK0Lt3b0yYMAGXLl2CJElYv349zGZzc9RLREREREQUEHpPf8FoNMJmsyE+Ph46nQ4mkwmxsbFwOBwYOXIknE4noqOjERMT0xz1EhERERGRhtRr7LokNXkcPAGA1WqF1Wq96bVnn30Wzz77rCpFERERERERaU2QCMAXCdPT05v7LYmIiIiINC8tLS3QJch2f6vOquafrf2HqvlyeD3bHhERERER0d0sIGee9CEdfFr+tQVTAQCvL1mtiYxd732hOGPotAF+q8MfGb/483HFGTVTHgEAnMz5X8UZnSY+BkA72+NOyli6dK3ijPnzJ/utjjsh43qOVjKWZKzxKWNByvOaWRdf+hS41qtaWRdmMIMZ3mf4Yz8252HJp4zl5SGa2B5O6ZRPywdCm58/omr+uYvKP5/6m9dnnpYtW4Z58+YBAA4cOICYmBiYTCZkZmaqVhwREREREZFWeDV4Ki4uxtatWwEAV69eRUpKCt555x3s3LkTR48exd69e1UtkoiIiIiItIn3ebrB+fPnkZmZienTpwMAbDYbOnXqhMjISOj1esTExGD37t2qF0pERERERBRIHqcqX7hwIWbNmgW73Q4AOHPmDNq2bdvw8/DwcFRVValXIRERERERadbddJ+nW5552rRpE9q3bw+DwdDwWn19PYKCghqeCyFuek5ERERERHQnuuWZp507d6K6uhpxcXG4cOECLl++jFOnTkGn0zX8nerqaoSHh6teKBERERERaY/WrktS0y0HTzk5OQ3/XVBQgNLSUqSnp8NkMuHkyZPo2LEjduzYgeHDh6teKBERERERUSB5vOapsRYtWuCNN96A1WqFw+GA0WjEkCFD1KiNiIiIiIg0rp5nnn4qISEBCQkJAACDwYDCwkLViiIiIiIiotuDuIsmjAgSAfiSYnp6enO/JRERERGR5qWlpQW6BNl+Fvagqvk/Xq5QNV+OgAye9CEdfFr+tQVTAQCvL1mtiYy/rv1accZvJz/htzr8kfGr944rzvh+2iMAgKqcbxVnREzsBkA720MrGUsy1ijOWJDyPADgzWXrFGe8OncCAO1sj0BnXM9hhv8zli5d61PG/PmTNbMuWsngNmXG3ZKRfPQLnzKyHx2giXVxSqd8Wj4QQkM7qZp/5cpJVfPl8HiTXCIiIiIiIpIxeFq2bBnmzZvX8Lyurg7jx4/HwYMHVSmMiIiIiIi0Twih6kNLvBo8FRcXY+vWrQ3PT5w4gbFjx+Kbb75RrTAiIiIiIiIt8Th4On/+PDIzMzF9+vSG1zZv3owpU6agR48eqhZHRERERETaJlT+oyUeB08LFy7ErFmz0KpVq4bX5syZg4EDB6paGBERERERkZbccvC0adMmtG/fHgaDobnqISIiIiKi28jddM3TLW+Su3PnTlRXVyMuLg4XLlzA5cuXkZGRgZSUlOaqj4iIiIiIyCvbt2/Hu+++C6fTifHjx2PMmDF+zb/l4CknJ6fhvwsKClBaWsqBExERERERNdDK2aGqqipkZmaioKAAISEhSEpKQu/evfGrX/3Kb+/B+zwREREREdFt78CBA+jTpw9at26NsLAwDB48GLt37/bre9zyzNONEhISkJCQcNNreXl5fi2GiIiIiIhuL2qfd6qtrUVtbe1PXm/VqtVNk9qdOXMGbdu2bXgeHh4Om83m11qChFbOsxERERERETWyYsUKrFy58ievz5gxA1arteH5u+++C4fDgZdffhkAkJ+fj6NHj2LRokV+q8XrM09ERERERETNbfz48Rg2bNhPXr/xrBMAtGvXDocOHWp4Xl1djfDwcL/WwsETERERERFpVuOv5zXld7/7HVasWIFz584hNDQUn3zyCV5//XW/1sLBExERERER3fYiIiIwa9YsjBs3DnV1dUhMTET37t39+h685omIiIiIiMgLnKqciIiIiIjICxw8EREREREReYGDJyIiIiIiIi9w8EREREREROQFDp6IiIiIiIi8oLnB0/bt22E2m2EymfDBBx8ozrl06RKio6NRWVmpaPmVK1fCYrHAYrFg+fLlijKysrJgNpthsViQk5OjKOO6ZcuWYd68eYqWHTt2LCwWC+Li4hAXF4cjR47Izvj888+RkJCAoUOHYvHixbKX37RpU8P7x8XFoVevXoru9rxt27aGf5dly5bJXh4AVq9ejcGDByMmJgbvvvuurGUb99WBAwcQExMDk8mEzMxMRRkAMGfOHBQUFCiuY+PGjYiOjkZMTAzmz58PSZJkZ3z44YewWCwwm81YtmwZvJmIs6nfs/Xr12Ps2LGK1mX+/PkwmUwNvfLpp5/Kzvjmm28wYsQIWCwWzJ49W/b22Lt3701HZIF7AAAO80lEQVT92qdPH0ybNk12Hfv27UNsbCyio6MxZ84cRf8uBQUFMJvNiImJweLFi+F0Om+5vLt9l9w+bWr/V1dXh/Hjx+PgwYOKMuT2qbsMuX16q325t33qLkNun7rLkNunjTOU9Km7OuT2qbsMuX3q7vgot0+bOsbK6VN3GXL71F2G3D691ecFb/vUXYbcPnWXIbdPG2co6VN3dcjtU3cZcvv0uhs/gyk57jfOAOT1KWmM0JB///vfYsCAAeI///mP+PHHH0VMTIw4fvy47Jy//e1vIjo6WnTr1k3861//kr38/v37xciRI4XD4RCSJIlx48aJTz75RFbGwYMHRVJSkqirqxNXrlwRAwYMEOXl5bJrEUKIAwcOiN69e4u5c+fKXra+vl707dtX1NXVKXpvIYT44YcfRN++fYXdbheSJIlRo0aJL7/8UnHeP/7xDzFo0CBRU1Mja7nLly+LJ554QtTU1Ii6ujqRmJgo9u/fLytj//79Ijo6Wly8eFE4nU4xbdo0sWfPHq+WbdxXV65cEUajUfzwww+irq5OTJo0yeN2aZzx73//W0ybNk10795dbNmyRVEdJ06cEIMGDRIXL14U9fX1Ys6cOSInJ0dWxg8//CAGDRokfvzxR+F0OsXIkSPFV199JSvjuuPHj4t+/fqJ5557Tva6CCFEdHS0qKqq8rwhmsi4ePGiePLJJ0VZWZkQQohZs2aJDz74QNG6CCHEmTNnxDPPPCP++c9/ys7o37+/+P7774UQQlitVpGfny8ro7y8XPTr169he6SlpYn333+/yeXd7bu2b98uq0+b2v+Vl5eLkSNHiscee0yUlJTccj3cZbz33nuy+tRdRk5Ojqw+vdW+3Ns+bSpDTp+6yygoKJDVp56OS970aVMZcvq0qX9bOX3q7vhYVlYmq0+bOsbK6dOmMuT06a0yvO3TW31e8LZPm8qQ06dN/bvI6VNPn3286dOmMuT0aVMZcvr0uhs/gyk57jfOEELI6lPSHk2deTpw4AD69OmD1q1bIywsDIMHD8bu3btl5+Tn5yMtLQ3h4eGK6mjbti3mzZuHkJAQBAcH4+GHH8bp06dlZURFRSE3Nxd6vR41NTVwuVwICwuTXcv58+eRmZmJ6dOny14WAE6cOAEAmDRpEmJjY7F+/XrZGZ9++inMZjPatWuH4OBgZGZmokePHorqAYDf//73mDVrFtq0aSNrOZfLhfr6ely5cgVOpxNOpxMtWrSQlfHdd9+hb9++aNmyJXQ6Hfr164fPPvvMq2Ub95XNZkOnTp0QGRkJvV6PmJgYj/3aOGP79u145plnMHToUK/XoXFGSEgI0tLS0LJlSwQFBaFz584e+7VxRmRkJD7++GOEhYWhtrYWly5d8ngnb3e/Z5IkYeHChUhOTla0LleuXMHp06eRkpKCmJgYZGdno76+XlbG/v370bNnT3Tp0gUAkJqaikGDBslel+uWL1+OpKQkPPjgg7IzXC4XLl26BJfLBYfD4bFfG2ccO3YMPXv2bHg+YMCAW/aru31XRUWFrD5tav+3efNmTJkyxavffXcZkiTJ6lN3GUFBQbL6tKl1kdOnTWXI6VN3GadOnZLVp56OS970aVMZcvq0qX9bOX3q7vhYW1srq0+bOsbK6VN3GS1atJDVp03VIadPm8qQ06fuMu69915Zfeouo6ysTFafevrs402fNpUhp0/dZdhsNll9Cvz0M5iS4767z3Fy+pS0R1ODpzNnzqBt27YNz8PDw1FVVSU7Z8mSJXj88ccV1/HII4+gZ8+eAICKigrs2rULRqNRdk5wcDCys7NhsVhgMBgQEREhO2PhwoWYNWuWxw+yTamtrYXBYMCqVauwbt06bNiwAfv375eVcfLkSbhcLkyfPh1xcXH48MMP8V//9V+K6jlw4ACuXr0qa7BwXcuWLTFz5kwMHToURqMRHTp0wG9/+1tZGd26dcO+fftw/vx5OBwOfP755zh79qxXyzbuKyX92jhjypQpePbZZ2WtQ+OMDh064MknnwQAnDt3Dh988AGeeeYZWRnAtX7Nz8/HwIED0bZt24aDpZyMt956C8OHD0dkZKSidTl79iz69OmDjIwM5Ofn49ChQ9i8ebOsjJMnTyIsLAyzZs1CXFwcVqxY4fH3p6l9RkVFBUpLSzFu3DjZ6wJc+x8FY8eORb9+/fCf//wHQ4YMkZXRpUsXHDlyBHa7HS6XC7t3775lv7rbdwUFBcnq06b2f3PmzMHAgQNvWf+tMqKjo2X1aVN1yOnTpjLk9Km7jH79+snqU3cZISEhsvr0Vsclb/u0qQw5feouw2w2y+pT4KfHRyX7U3fHWDl96i7jl7/8pez9qbs65O5P3WXI3Z82znA6nbL3p40zqqurZe9Pm/rsI2d/6i5D7v60cUb37t1l92njz2BK+tTd5zi5fUraoqnBU319PYKCghqeCyFuet7cjh8/jkmTJmHOnDke/69zU5KTk1FcXAy73Y78/HxZy27atAnt27eHwWBQ9N4A8Jvf/AbLly/Hz3/+c7Rp0waJiYnYu3evrAyXy4Xi4mJkZGRg48aNsNls2Lp1q6J6NmzYgIkTJypa9u9//zu2bNmCL774Al999RXuuecerF27VlaGwWBAQkICxo4diylTpqBXr14IDg5WVI/W+rWqqgrjx4/H8OHD0bt3b0UZI0aMwMGDB3H//fdj5cqVspbdv38/7HY7hg8frui9gWtnwFatWoXw8HCEhoZi7Nixivp13759mD17NgoKCnDlyhWsXr1aUT0bN27E6NGjERISInvZ6upq/OEPf8COHTuwb98+9OjRA0uXLpWV8d///d945ZVX8MILL2DMmDH49a9/7VW/3rjvioyMVNSn/tj/ucuQ26fuMuT26Y0Zp06dUtSnN2Y89NBDivr0xgylfepue8jt0xszfvaznynq08bbQ0mf3nh8rKioUNSnvhxjb5Uht0/dZcjt0xszNm7cqKhPb8woLi5W1Kc3ZkiSpKhP3W0PuX16Y8aqVasU9emNGV9//bWsPnX3GUzucd8fn+NIezQ1eGrXrh2qq6sbnldXVyv+6p2vDh8+jAkTJuCVV17BsGHDZC9fXl6OsrIyAEBoaChMJhOOHTsmK2Pnzp3Yv38/4uLikJ2djc8//xwZGRmyMg4dOoTi4uKG50II6PV6WRn3338/DAYD2rRpg3vvvRcDBw6EzWaTlQFc+0rX119/jaefflr2ssC1i0UNBgN+8YtfICQkBAkJCSgtLZWVcenSJZhMJmzfvh15eXkICQnx+v/qNaalfi0vL0dSUhKGDRuGl156Sfbydrsdhw8fBgDo9XpYLBbZ/bpjxw4cP34ccXFxSE1NxdGjR/Hyyy/Lyjh27Bj27NnT8Fxpv/bo0QORkZHQ6XQYOnSoon4FgKKiIpjNZkXLHjp0CJ07d8YDDzyAe+65ByNGjJDdrw6HA927d8dHH32EDRs2ICIiwmO/Nt53KelTX/d/TWXI7dPGGUr6tHGGkj5tnKGkTxtnKOnTpv5d5PRp4wwlfdo4Q26fujs+Hjx4UFaf+uMY21SGnD51l3HkyBFZfdpUhpw+dZexc+dOWX3qLmP16tWy+vRW/y7e9qm7jF27dsnqU3cZNptNVp+6+wy2adMmWX3qj89xpEEBu9rKjesTRtTU1IjLly+L2NhYceTIEcV5AwYMUDRhxOnTp0Xv3r3FgQMHFL/3l19+KRISEoTD4RAOh0NMnDhR7NixQ3Heli1bFE0Y8fnnn4v4+Hhx9epVcfHiRRETEyP++te/ysr429/+JgYPHiwuXLjQMMmCpwvf3bHZbCIpKUn2ctd99dVXIjY2Vvz444+ivr5evPbaayI7O1tWRllZmYiNjRV1dXWitrZWDB48WBw6dEhWxvW+unr1qujfv7+oqKgQTqdTTJ48WezcuVNWxnVz5871esKIxhkXL14URqNRbN26VdbyN2YcO3ZMDBgwQFy4cEHU19eLefPmiffee09Wxo1KSkq8mjCicUZZWZno37+/OH/+vJAkSUyaNEls375dVsbp06dFv379xOnTp4UQ1y4KzszMlL0uNTU1ol+/fl6vQ+OM77//XhiNRlFdXS2EEOLdd9/1+nf4esa5c+eE0WgUFy9eFA6HQ4wePVoUFhY2uZy7fZfcPvW0/3vuuec8XuDsLkNun7rLkNunntbFmz51lyG3T91lyO3TptZFTp+6y5Dbp+4y5PZpU8dHOX3q6RjrTZ+6y9i6dausPnWXsWrVKll96mldvOlTdxkrVqyQ1adNbQ85fdrUusjpU3cZf/rTn2T1qbuMbdu2yerTG13/DObLcd/d5zhv+pS0R97/0lVZREQEZs2ahXHjxqGurg6JiYno3r17s9exdu1aOBwOvPHGGw2vJSUlYdSoUV5nGI1G2Gw2xMfHQ6fTwWQywWKxqFHuLQ0YMABHjhxBfHw86uvrMXr0aPzmN7+RldGjRw9MmTIFo0ePRl1dHZ588klFX83617/+hXbt2sle7rq+ffviu+++Q0JCAoKDg/HYY49h6tSpsjK6dOkCk8mE2NhYuFwuTJgwAb169VJUT4sWLfDGG2/AarXC4XDAaDR6/A62GjZv3oyzZ88iJyenYTrWp59+GjNnzvQ6o3Pnzpg6dSqSkpKg0+nw+OOPK/56pS+6dOmCqVOnYtSoUXA6nTCZTIiOjpaV0b59eyxatAjTp0+Hw+FA165dMXfuXNm1VFZW+tSvDz/8MGbOnIlx48ZBp9OhU6dOsqfnv++++/DSSy9h5MiRcDqdDdMnN6WpfZecPvXH/s9dhtlsltWnTdUhp0/VWpfrdXjbp01lyOnTpjK6devmdZ82lSGnT5vKkNOnTR0f27Rp43Wf+uMY6y7j/PnzsvrUXcaLL76INm3aeN2naq3LjBkzcN9993ndp+4y4uPj0bp1a6/7tKl1sdlsXvepu4xp06YhPDzc6z51lxEbGwuHw+F1n7qjleM+BVaQEF7czIWIiIiIiOgup6lrnoiIiIiIiLSKgyciIiIiIiIvcPBERERERETkBQ6eiIiIiIiIvMDBExERERERkRc4eCIiIiIiIvICB09ERERERERe4OCJiIiIiIjIC/8fdYJhUevHLecAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x648 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (16,9))\n",
    "sns.heatmap(matrix, linecolor='grey', linewidth=0.1)\n",
    "plt.savefig('confusion_matrix.png', \n",
    "            #transparent=True,\n",
    "            bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_smoother(x, factor=0.1):\n",
    "    xx = x.copy()\n",
    "    xx[x == 1] = 1 - factor\n",
    "    xx[x == 0] = factor / (y_true.shape[1] - 1)\n",
    "    return xx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss: 5.8365355\n",
      "Accuracy: 0.03046875\n"
     ]
    }
   ],
   "source": [
    "loss     = tfk.losses.categorical_crossentropy(y_pred, label_smoother(y_true)).numpy().mean()\n",
    "accuracy = accuracy_score(y_pred.argmax(axis=1), y_true.argmax(axis=1))\n",
    "\n",
    "print(\"Loss:\", loss)\n",
    "print(\"Accuracy:\", accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30694</th>\n",
       "      <td>0000ffaa6ce77e3dde38bcefa039a4a4.jpg</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50947</th>\n",
       "      <td>00017f8bf9d8a34dab8f9707a378dbe5.jpg</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31622</th>\n",
       "      <td>00022314f9c58eb20d248ed239db3a52.jpg</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34234</th>\n",
       "      <td>0002d098348d8358c44032903e117ade.jpg</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12787</th>\n",
       "      <td>000367a6ec58e9d1e4bc340d8d871dab.jpg</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   filename  category\n",
       "30694  0000ffaa6ce77e3dde38bcefa039a4a4.jpg        16\n",
       "50947  00017f8bf9d8a34dab8f9707a378dbe5.jpg        38\n",
       "31622  00022314f9c58eb20d248ed239db3a52.jpg        16\n",
       "34234  0002d098348d8358c44032903e117ade.jpg        28\n",
       "12787  000367a6ec58e9d1e4bc340d8d871dab.jpg        34"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_category_losses(category_i):\n",
    "    pd_cat_i = pd_train[pd_train['category'] == category_i]\n",
    "    pd_cat_i = pd_cat_i.sort_values(by=['filename'])\n",
    "    \n",
    "    cat_i_datagen = tfk_ImageDataGenerator(\n",
    "        preprocessing_function=tfk_preprocess_input\n",
    "    )\n",
    "\n",
    "    cat_i_generator = cat_i_datagen.flow_from_dataframe(\n",
    "        pd_cat_i,\n",
    "        directory='shopee-product-detection-dataset/train/{:02d}'.format(category_i),\n",
    "        x_col='filename',\n",
    "        y_col='category',\n",
    "        target_size=input_shape[:2],\n",
    "        color_mode='rgb',\n",
    "        batch_size=batch_size,\n",
    "        class_mode='raw',\n",
    "        shuffle=False,\n",
    "    )\n",
    "    \n",
    "    y_pred_i = model.predict(cat_i_generator, verbose=1, steps=len(cat_i_generator.labels)//batch_size+1)\n",
    "    y_pred_i = y_pred_i[:len(cat_i_generator.labels)]\n",
    "    \n",
    "    y_true_i = np.full((len(cat_i_generator.labels), 42), 0.1/41, dtype='float')\n",
    "    y_true_i[:,category_i] = 0.9\n",
    "    \n",
    "    cce = tfk.losses.CategoricalCrossentropy(reduction=tfk.losses.Reduction.NONE)\n",
    "    losses = cce(y_pred_i, y_true_i).numpy()\n",
    "    return pd_cat_i, y_pred_i, losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_train_losses = []\n",
    "for i in range(42):\n",
    "    pd_cat_i, y_pred_i, losses = calc_category_losses(i)\n",
    "    pd_cat_i['predicted_category'] = y_pred_i.argmax(axis=1)\n",
    "    pd_cat_i['loss'] = losses\n",
    "    pd_train_losses.append(pd_cat_i)\n",
    "pd_train_losses = pd.concat(pd_train_losses, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "      <th>predicted_category</th>\n",
       "      <th>loss</th>\n",
       "      <th>cum_mistakes</th>\n",
       "      <th>mistake</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16f9a736b9bbccc55447ee55019c5912.jpg</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>6.015628</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a26df05f3d18d5dbb4027707d3a81168.jpg</td>\n",
       "      <td>4</td>\n",
       "      <td>29</td>\n",
       "      <td>6.014174</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75f6d645f06733d735d642446c135076.jpg</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>6.013391</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0d6481d48dfa9689ef6aa020cd9e5b94.jpg</td>\n",
       "      <td>29</td>\n",
       "      <td>4</td>\n",
       "      <td>6.013052</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>011ef33346d01259f627b44bc5060584.jpg</td>\n",
       "      <td>41</td>\n",
       "      <td>22</td>\n",
       "      <td>6.012937</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               filename  category  predicted_category  \\\n",
       "0  16f9a736b9bbccc55447ee55019c5912.jpg        12                  26   \n",
       "1  a26df05f3d18d5dbb4027707d3a81168.jpg         4                  29   \n",
       "2  75f6d645f06733d735d642446c135076.jpg         0                   4   \n",
       "3  0d6481d48dfa9689ef6aa020cd9e5b94.jpg        29                   4   \n",
       "4  011ef33346d01259f627b44bc5060584.jpg        41                  22   \n",
       "\n",
       "       loss  cum_mistakes  mistake  \n",
       "0  6.015628             1        1  \n",
       "1  6.014174             2        1  \n",
       "2  6.013391             3        1  \n",
       "3  6.013052             4        1  \n",
       "4  6.012937             5        1  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_train_losses = pd_train_losses.sort_values(by=['loss'], ascending=False)\n",
    "pd_train_losses = pd_train_losses.reset_index().drop('index', axis=1)\n",
    "pd_train_losses['mistake'] = 1*(pd_train_losses['category'] != pd_train_losses['predicted_category'])\n",
    "pd_train_losses['cum_mistakes'] = pd_train_losses['mistake'].cumsum()\n",
    "pd_train_losses.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAD+CAYAAADGZlkKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAXN0lEQVR4nO3df0xV9x3/8detF4gZpPva3DsNtW7tTEwwEzaS6dJdYheBDu5YickUBmvcpjHWrdZiEJmEzV8xVDfrj/jHkmZqMhktXCV4qUk3k06zIdk0NtS5TJyCwwsy5VLBy72f7x/N7reo/fIB4V6B5+MfPYdz/Zw3h95nOLcXHMYYIwAALDwV7xMAAEweRAMAYI1oAACsEQ0AgDWiAQCwRjQAANacNgft379fp06dkiRlZWVp06ZN2rx5s1pbWzVz5kxJ0muvvaZly5apra1NW7ZsUX9/vzIzM1VdXS2n06nOzk6VlZWpp6dHX/nKV1RTU6MvfOELEzcZAGDcjfidxtmzZ/Xhhx+qvr5eDQ0N+uijj3T69GldunRJR48elc/nk8/n07JlyyRJZWVl2rp1q5qbm2WMUW1trSSpurpaRUVF8vv9WrhwoQ4ePDixkwEAxt2I0XC5XCovL1diYqISEhL0wgsvqLOzU52dnaqoqJDX69W+ffsUiUTU0dGhgYEBpaenS5IKCwvl9/sVCoXU0tKinJycYfsBAJPLiLen5s+fH/17e3u7Tp06pWPHjumvf/2rqqqqlJKSojVr1qiurk7z58+Xy+WKHu9yudTV1aXe3l4lJyfL6XQO2w8AmFysXwi/cuWKVq1apU2bNun555/XgQMH5Ha7NXPmTJWUlOjMmTOKRCJyOBzRxxhj5HA4on9+1oPbAIAnn9UL4a2trfrZz36miooK5eXl6fLly2pvb4/ebjLGyOl0avbs2QoEAtHHdXd3y+12a9asWerr61M4HNaMGTMUCATkdrtHdaI9PUFFIqP/MVkuV4oCgb5RP26qYH7mn67zT+fZpU/nnwgjfqdx8+ZNrVu3TjU1NcrLy5P0aSR27NihO3fuKBQK6fjx41q2bJlSU1OVlJSk1tZWSZLP55PH41FCQoIyMzPV1NQkSWpoaJDH45mQgQAAE8cx0k+53bZtm959910999xz0X0rVqxQJBLRsWPHNDQ0pOzsbL355puSpI8//liVlZUKBoNKS0vTzp07lZiYqI6ODpWXl6unp0dz5szRnj179PTTT1ufKN9pjA3zM/90nX86zy5N3HcaI0bjSUE0xob5mX+6zj+dZ5fieHsKAID/IRoAAGtEAwBgjWgAAKxZvU9jMrsfCk/YC0IjGbwfVlLijLivG8v5BwaH1Hf3XszWAxBbUz4aiQkz5N3oi8vaJ98qiMva8Vr3f2tP3/9fBZj6uD0FALBGNAAA1qb87SnEVrxeQ+K1FCA2iAbGVbxeQ+K1FCA2uD0FALBGNAAA1ogGAMAa0QAAWCMaAABrRAMAYI1oAACsEQ0AgDWiAQCwRjQAANaIBgDAGtEAAFgjGgAAa0QDAGCNaAAArBENAIA1ogEAsEY0AADWiAYAwBq/IxxTwv1QWC5XyiM/9nn7x8vA4JD67t6b0DWAJwXRwJSQmDBD3o2+uKx98q0C9cVlZSD2uD0FALBGNAAA1ogGAMAa0QAAWLOKxv79+5WXl6e8vDzt3r1bknT27Fl5vV5lZ2dr79690WPb2tpUWFionJwcbdmyRUNDQ5Kkzs5OFRcXKzc3V2vXrlV/f/8EjAMAmEgjRuPs2bP68MMPVV9fr4aGBn300UdqbGxURUWFDh48qKamJl26dElnzpyRJJWVlWnr1q1qbm6WMUa1tbWSpOrqahUVFcnv92vhwoU6ePDgxE4GABh3I0bD5XKpvLxciYmJSkhI0AsvvKD29nbNmzdPc+fOldPplNfrld/vV0dHhwYGBpSeni5JKiwslN/vVygUUktLi3JycobtBwBMLiO+T2P+/PnRv7e3t+vUqVP64Q9/KJfLFd3vdrvV1dWlW7duDdvvcrnU1dWl3t5eJScny+l0Dts/Gs88kzyq44FYmug3ED6OJ/ncJtp0nn2iWL+578qVK1qzZo02bdqkGTNmqL29PfoxY4wcDocikYgcDsdD+//352c9uD2Snp6gIhEzqsdIfNEgNgKBJ/PtfS5XyhN7bhNtOs8uTdxzn9UL4a2trXr11Ve1ceNGvfLKK5o9e7YCgUD044FAQG63+6H93d3dcrvdmjVrlvr6+hQOh4cdDwCYXEaMxs2bN7Vu3TrV1NQoLy9PkrRo0SJdvXpV165dUzgcVmNjozwej1JTU5WUlKTW1lZJks/nk8fjUUJCgjIzM9XU1CRJamhokMfjmcCxAAATYcTbU7/97W81ODioXbt2RfetWLFCu3bt0vr16zU4OKisrCzl5uZKkmpqalRZWalgMKi0tDSVlpZKkqqqqlReXq5Dhw5pzpw52rNnzwSNBACYKCNGo7KyUpWVlY/82IkTJx7at2DBAtXV1T20PzU1VUeOHBnDKQIAnhS8IxwAYI1oAACsEQ0AgDWiAQCwRjQAANaIBgDAGtEAAFgjGgAAa0QDAGCNaAAArBENAIA1ogEAsEY0AADWiAYAwBrRAABYs/4d4QAe7X4oHJffRT8wOKS+u/divi6mN6IBPKbEhBnybvTFfN2TbxWoL+arYrrj9hQAwBrRAABYIxoAAGtEAwBgjWgAAKwRDQCANaIBALBGNAAA1ogGAMAa0QAAWCMaAABrRAMAYI1oAACsEQ0AgDWiAQCwRjQAANaIBgDAmlU0gsGg8vPzdePGDUnS5s2blZ2drYKCAhUUFOj06dOSpLa2NhUWFionJ0dbtmzR0NCQJKmzs1PFxcXKzc3V2rVr1d/fP0HjAAAm0ojRuHDhglauXKn29vbovkuXLuno0aPy+Xzy+XxatmyZJKmsrExbt25Vc3OzjDGqra2VJFVXV6uoqEh+v18LFy7UwYMHJ2YaAMCEGjEatbW1qqqqktvtliTdu3dPnZ2dqqiokNfr1b59+xSJRNTR0aGBgQGlp6dLkgoLC+X3+xUKhdTS0qKcnJxh+wEAk49zpAO2b98+bLu7u1uLFy9WVVWVUlJStGbNGtXV1Wn+/PlyuVzR41wul7q6utTb26vk5GQ5nc5h+0frmWeSR/0YYKpzuVLG5ZipajrPPlFGjMaD5s6dqwMHDkS3S0pK1NDQoBdeeEEOhyO63xgjh8MR/fOzHty20dMTVCRiRv04vmgwlQUCff/fj7tcKSMeM1VN59mliXvuG/X/PXX58mU1NzdHt40xcjqdmj17tgKBQHR/d3e33G63Zs2apb6+PoXDYUlSIBCI3uoCAEwuo46GMUY7duzQnTt3FAqFdPz4cS1btkypqalKSkpSa2urJMnn88nj8SghIUGZmZlqamqSJDU0NMjj8YzvFACAmBj17akFCxZo9erVWrlypYaGhpSdna38/HxJUk1NjSorKxUMBpWWlqbS0lJJUlVVlcrLy3Xo0CHNmTNHe/bsGd8pAAAxYR2NDz74IPr34uJiFRcXP3TMggULVFdX99D+1NRUHTlyZIynCAB4UvCOcACANaIBALBGNAAA1ogGAMAa0QAAWCMaAABrRAMAYI1oAACsEQ0AgDWiAQCwNuqfPQXgyXA/FI7L79MYGBxS39174/pvYvIgGsAklZgwQ96Nvpive/KtAk3f31IBbk8BAKwRDQCANaIBALBGNAAA1ogGAMAa0QAAWCMaAABrRAMAYI1oAACsEQ0AgDWiAQCwRjQAANaIBgDAGtEAAFgjGgAAa0QDAGCNaAAArBENAIA1ogEAsEY0AADWiAYAwBrRAABYs4pGMBhUfn6+bty4IUk6e/asvF6vsrOztXfv3uhxbW1tKiwsVE5OjrZs2aKhoSFJUmdnp4qLi5Wbm6u1a9eqv79/AkYBAEy0EaNx4cIFrVy5Uu3t7ZKkgYEBVVRU6ODBg2pqatKlS5d05swZSVJZWZm2bt2q5uZmGWNUW1srSaqurlZRUZH8fr8WLlyogwcPTtxEAIAJM2I0amtrVVVVJbfbLUm6ePGi5s2bp7lz58rpdMrr9crv96ujo0MDAwNKT0+XJBUWFsrv9ysUCqmlpUU5OTnD9gMAJh/nSAds37592PatW7fkcrmi2263W11dXQ/td7lc6urqUm9vr5KTk+V0OoftBwBMPiNG40GRSEQOhyO6bYyRw+H43P3/+/OzHty28cwzyaN+DICJ4XKlxPsUrEyW85xMRh2N2bNnKxAIRLcDgYDcbvdD+7u7u+V2uzVr1iz19fUpHA5rxowZ0eNHq6cnqEjEjPpxfNEA4y8Q6Iv3KYzI5UqZFOc5USbquW/U/8vtokWLdPXqVV27dk3hcFiNjY3yeDxKTU1VUlKSWltbJUk+n08ej0cJCQnKzMxUU1OTJKmhoUEej2d8pwAAxMSov9NISkrSrl27tH79eg0ODiorK0u5ubmSpJqaGlVWVioYDCotLU2lpaWSpKqqKpWXl+vQoUOaM2eO9uzZM75TAIiZ+6Fw3L6DHxgcUt/de3FZG5+yjsYHH3wQ/fuSJUt04sSJh45ZsGCB6urqHtqfmpqqI0eOjPEUATxJEhNmyLvRF5e1T75VoOl7w+nJwDvCAQDWiAYAwBrRAABYIxoAAGtEAwBgjWgAAKwRDQCANaIBALBGNAAA1ogGAMAa0QAAWCMaAABrRAMAYI1oAACsEQ0AgDWiAQCwRjQAANaIBgDAGtEAAFgjGgAAa0QDAGCNaAAArBENAIA1ogEAsEY0AADWiAYAwBrRAABYIxoAAGtEAwBgjWgAAKwRDQCANaIBALBGNAAA1ogGAMAa0QAAWHM+zoNLSkp0+/ZtOZ2f/jO//OUv1d/fr507d2pwcFAvv/yyNmzYIElqa2vTli1b1N/fr8zMTFVXV0cfBwCYHMb8rG2MUXt7u/74xz9Gn/wHBgaUm5urI0eOaM6cOVqzZo3OnDmjrKwslZWVadu2bUpPT1dFRYVqa2tVVFQ0boMAACbemG9P/etf/5IkrVq1St/73vd09OhRXbx4UfPmzdPcuXPldDrl9Xrl9/vV0dGhgYEBpaenS5IKCwvl9/vHZwIAQMyM+TuNu3fvasmSJfrFL36hUCik0tJS/eQnP5HL5Yoe43a71dXVpVu3bg3b73K51NXVNar1nnkmeaynCmCKuB8Ky+VKsT5+NMeOtG5iwoxx+bcmuzFHIyMjQxkZGdHt5cuXa9++ffrGN74R3WeMkcPhUCQSkcPheGj/aPT0BBWJmFGf53h90QCIv8SEGfJu9MV83ZNvFSgQ6Iv5uo9jop77xnx76vz58zp37lx02xij1NRUBQKB6L5AICC3263Zs2cP29/d3S232z3WpQEAcTLmaPT19Wn37t0aHBxUMBhUfX293njjDV29elXXrl1TOBxWY2OjPB6PUlNTlZSUpNbWVkmSz+eTx+MZtyEAALEx5ttTS5cu1YULF/T9739fkUhERUVFysjI0K5du7R+/XoNDg4qKytLubm5kqSamhpVVlYqGAwqLS1NpaWl4zYEACA2HuuNEq+//rpef/31YfuWLFmiEydOPHTsggULVFdX9zjLAQDijHeEAwCsEQ0AgDWiAQCwRjQAANaIBgDAGtEAAFgjGgAAa0QDAGCNaAAArBENAIA1ogEAsEY0AADWiAYAwBrRAABYIxoAAGtEAwBgjWgAAKwRDQCANaIBALBGNAAA1ogGAMAa0QAAWCMaAABrRAMAYI1oAACsEQ0AgDWiAQCwRjQAANaIBgDAGtEAAFhzxvsEAOBJdz8UlsuVEpe1BwaH1Hf3XlzWfhSiAQAjSEyYIe9GX1zWPvlWgfrisvKjcXsKAGCNaAAArMU0GidPntR3v/tdZWdn69ixY7FcGgAwDmL2mkZXV5f27t2r9957T4mJiVqxYoW++c1v6qtf/WqsTgEA8JhiFo2zZ89q8eLF+uIXvyhJysnJkd/v12uvvWb1+Keecox5bff/mTnmxz6ueK3NzNNj7em2bjzXjufMj/P8N94cxhgTi4UOHz6sTz75RBs2bJAk/eEPf9DFixf1q1/9KhbLAwDGQcxe04hEInI4/l8tjTHDtgEAT76YRWP27NkKBALR7UAgILfbHavlAQDjIGbR+Na3vqVz587p9u3bunfvnt5//315PJ5YLQ8AGAcxeyH8S1/6kjZs2KDS0lKFQiEtX75cX/va12K1PABgHMTshXAAwOTHO8IBANaIBgDAGtEAAFgjGgAAa1M6GlP1BySWlJQoLy9PBQUFKigo0IULF3T27Fl5vV5lZ2dr79690WPb2tpUWFionJwcbdmyRUNDQ5Kkzs5OFRcXKzc3V2vXrlV/f3+8xrEWDAaVn5+vGzduSNK4zXz37l2tXr1aL7/8soqLi4e9n+hJ8uD8mzdvVnZ2dvTr4PTp05Km3vz79+9XXl6e8vLytHv3bknT69o/av64XnszRf3nP/8xS5cuNb29vaa/v994vV5z5cqVeJ/WY4tEIubFF180oVAouu/evXsmKyvL/Pvf/zahUMisWrXK/OlPfzLGGJOXl2f+9re/GWOM2bx5szl27JgxxpjVq1ebxsZGY4wx+/fvN7t3747xJKPz97//3eTn55u0tDRz/fr1cZ25urraHD582BhjTH19vfn5z38e6/FG9OD8xhiTn59vurq6Hjp2Ks3/5z//2fzgBz8wg4OD5v79+6a0tNScPHly2lz7R83//vvvx/XaT9lovPfee2bz5s3R7f3795u33347jmc0Pv75z3+aF1980ZSUlBiv12uOHDli/vKXv5jS0tLoMfX19aa8vNzcuHHDfOc734nub2lpMSUlJeb+/fsmIyMjGp7Ozk7z0ksvxXyW0aioqDAtLS1m6dKl5vr16+M689KlS01nZ6cxxphQKGQyMjLM/fv3YzjdyB6c/5NPPjFf//rXzY9//GOTn59vfvOb35hwODzl5v/HP/4RfRI05tMnubfffnvaXPtHzf/OO+/E9dpP2dtTt27dksvlim673W51dXXF8YzGx927d7VkyRIdOHBA77zzjn7/+9+rs7PzkbM++DlwuVzq6upSb2+vkpOT5XQ6h+1/km3fvl2ZmZnR7c+7vmOZ+bOPcTqdSk5O1u3bt2MxlrUH5+/u7tbixYu1Y8cO1dbW6vz586qrq5ty88+fP1/p6emSpPb2dp06dUoOh2PaXPtHzf/tb387rtd+ykZjqv6AxIyMDO3evVspKSmaNWuWli9frn379j1y1s/7HDzqczHZPjefN9t4zGyM0VNPPdn/acydO1cHDhyQ2+3WzJkzVVJSojNnzkzZ+a9cuaJVq1Zp06ZNmjt37rS79p+d//nnn4/rtX/yPjvjZKr+gMTz58/r3Llz0W1jjFJTUx8564Ofg+7ubrndbs2aNUt9fX0Kh8PDjp9MPu/6jmVmt9ut7u5uSdLQ0JD6+/ujv/flSXX58mU1NzdHt40xcjqdU3L+1tZWvfrqq9q4caNeeeWVaXftH5w/3td+ykZjqv6AxL6+Pu3evVuDg4MKBoOqr6/XG2+8oatXr+ratWsKh8NqbGyUx+NRamqqkpKS1NraKkny+XzyeDxKSEhQZmammpqaJEkNDQ2T7nOzaNGicZs5KytLDQ0NkqSmpiZlZmYqISEhPoNZMsZox44dunPnjkKhkI4fP65ly5ZNuflv3rypdevWqaamRnl5eZKm17V/1PzxvvZT+mdPnTx5UocPH47+gMSf/vSn8T6lcfHrX/9azc3NikQiKioq0o9+9COdO3dOO3fu1ODgoLKysrR582Y5HA59/PHHqqysVDAYVFpamnbu3KnExER1dHSovLxcPT09mjNnjvbs2aOnn3463qON6KWXXtLvfvc7Pfvss+M283//+1+Vl5fr+vXrSklJUU1NjZ599tl4j/pIn53/2LFjOnbsmIaGhpSdna0333xTkqbU/Nu2bdO7776r5557LrpvxYoV+vKXvzwtrv3nzR+JROJ27ad0NAAA42vK3p4CAIw/ogEAsEY0AADWiAYAwBrRAABYIxoAAGtEAwBgjWgAAKz9X0n6sUyd2s0AAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(pd_train_losses[pd_train_losses['mistake'] == 1].index);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_mistakes   = pd_train_losses['cum_mistakes'].max()\n",
    "mistakes_ratio = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "      <th>predicted_category</th>\n",
       "      <th>loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23867</th>\n",
       "      <td>2b4e38b1276d74ec78ed07e48f43b0a0.jpg</td>\n",
       "      <td>38</td>\n",
       "      <td>38</td>\n",
       "      <td>3.188401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23868</th>\n",
       "      <td>03a368b5e2a9520cb13547f888544190.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.188327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23869</th>\n",
       "      <td>1ef0c3aea58a39299b999c7a3193bd3c.jpg</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.187934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23870</th>\n",
       "      <td>fa84c252694018edd44d751d1c5e649f.jpg</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>3.187856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23871</th>\n",
       "      <td>ca3f914c7a019fc0477426c4d8b3e351.jpg</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>3.187367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23872</th>\n",
       "      <td>d4672a0e7a79d464a7c90fa2218a1132.jpg</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>3.187084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23873</th>\n",
       "      <td>92ae03813b87210e4ae49c0099cfb4da.jpg</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>3.187005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23874</th>\n",
       "      <td>0e9f5bf576df19b3df5e361cf9462a1c.jpg</td>\n",
       "      <td>41</td>\n",
       "      <td>41</td>\n",
       "      <td>3.186953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23875</th>\n",
       "      <td>b86235e5f4a6a60211ffb6672ac326d4.jpg</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>3.186615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23876</th>\n",
       "      <td>97b79043fbeedacb40b4e2aef3f70e1b.jpg</td>\n",
       "      <td>35</td>\n",
       "      <td>35</td>\n",
       "      <td>3.186590</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   filename  category  predicted_category  \\\n",
       "23867  2b4e38b1276d74ec78ed07e48f43b0a0.jpg        38                  38   \n",
       "23868  03a368b5e2a9520cb13547f888544190.jpg         1                   1   \n",
       "23869  1ef0c3aea58a39299b999c7a3193bd3c.jpg         1                   1   \n",
       "23870  fa84c252694018edd44d751d1c5e649f.jpg        11                  11   \n",
       "23871  ca3f914c7a019fc0477426c4d8b3e351.jpg        24                  24   \n",
       "23872  d4672a0e7a79d464a7c90fa2218a1132.jpg         9                   9   \n",
       "23873  92ae03813b87210e4ae49c0099cfb4da.jpg        10                  10   \n",
       "23874  0e9f5bf576df19b3df5e361cf9462a1c.jpg        41                  41   \n",
       "23875  b86235e5f4a6a60211ffb6672ac326d4.jpg         8                   8   \n",
       "23876  97b79043fbeedacb40b4e2aef3f70e1b.jpg        35                  35   \n",
       "\n",
       "           loss  \n",
       "23867  3.188401  \n",
       "23868  3.188327  \n",
       "23869  3.187934  \n",
       "23870  3.187856  \n",
       "23871  3.187367  \n",
       "23872  3.187084  \n",
       "23873  3.187005  \n",
       "23874  3.186953  \n",
       "23875  3.186615  \n",
       "23876  3.186590  "
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_confusing_images = pd_train_losses[pd_train_losses['cum_mistakes'] <= num_mistakes*mistakes_ratio-EPS]\n",
    "pd_confusing_images = pd_confusing_images.drop(['cum_mistakes', 'mistake'], axis=1)\n",
    "pd_confusing_images.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22655419766206164"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd_confusing_images) / len(pd_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_confusing_images.to_csv('shopee-product-detection-dataset/confusing_images_2__mistake_ratio={}.csv'.format(mistakes_ratio), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8AAAAIzCAYAAADPp3CYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeWBMV+P/8c9ksyUEjaWxFJXEltj3BqngaaQEbe1ttapKdVFbUUtttXWhQeljieWhUWuJ1r59aSna2h5aOyVUJBGy3t8ffplHREhikoj7fv3Tzplzzz33zkzMZ86551oMwzAEAAAAAMATzi6nOwAAAAAAQHYgAAMAAAAATIEADAAAAAAwBQIwAAAAAMAUCMAAAAAAAFMgAAMAAAAATIEADADpNG3aNHl6eqpbt25K6w5ykZGR1jo5JbmfGzduzLE+ZEZCQoI+++wzNWrUSNWqVVNgYGC6tvv555/Vv39/+fv7q1q1aqpTp466d++u1atXp/k6ZYWYmBgNGTJE9erVk7e3t3r16pUl+9m7d688PT01duzYLGk/PZLfY99//32O9QHpd/r0aX322Wdq3bq1atSooerVqysoKEizZs1STExMirrnz5+Xp6en3nnnnRzqLQBkLYec7gAA5DY///yzQkND9dJLL+V0V54ooaGh+ve//61y5copKChIRYsWfWD9uLg4jR49Wt99950KFCigJk2aqEWLFvrnn3+0efNmDRgwQFu3btXkyZNlZ5f1v/fOmDFD33//vapWraqGDRuqXLlyWbIfd3d39e3bVz4+PlnSPp4sixcv1rhx45SYmKhGjRqpUaNGun37tn7++WdNnTpVq1evVkhIiIoUKZLTXQWAbEEABoBMmDRpkpo1a6annnoqp7vyxDhy5Igk6ZNPPlHDhg0fWn/UqFEKDQ2Vn5+fxo8fL1dXV+tz0dHR6tOnj3744QeVLFlSAwYMyLJ+J0vu/9SpU1W2bNks20+pUqX07rvvZln7eHKsWrVKo0aNUtmyZTVjxgxVqFDB+lxiYqI+//xzzZ49W2+//baWLVuWgz0FgOzDFGgAyKDKlSvrxo0bGjNmTE535YkSFxcnSSpcuPBD6+7Zs0ehoaGqWLGivvzyyxThV5KcnZ315ZdfKn/+/Fq0aJGuX7+eJX2+W0b6D2S1yMhIffrpp3J0dNS3336bIvxKkr29vT766CPVqFFDhw4d0vbt23OopwCQvQjAAJBBPXv2VLly5bR+/Xpt2bLlofW///57eXp6at68eame69atmzw9PRUZGSnpf9ffBQcH68cff1RQUJC8vb3l5+enuXPnSpL279+vzp07q3r16vLz89O0adOUkJCQqu3bt29r3LhxatCggapXr65u3bpp79699+3j+vXr1bFjR9WoUUM1a9bUq6++qj179qSok3zt6eLFi/Xhhx/K29tbjRs31v79+x94/Lt27dLrr7+umjVrytvbW0FBQVq0aJGSkpJSHPOKFSskSW3btpWnp2eafZXuTJeWpDfffFNOTk73rePq6qqRI0dq7NixcnR0tJYnJSVp8eLFatu2rby9vVWrVi29/vrr2rVrV4rtk/s1bdo0bdq0SR06dJC3t7caNGigYcOG6Z9//klxXn7++WdJUp06deTp6anz588/8HpsPz8/1a5dO0XZzp079eqrr6pBgwby8fFRYGCgZs2aZQ3Xd+/v7muAu3XrJj8/P/3999/q37+/6tWrJx8fH3Xp0uW+5/Hs2bP68MMP1bBhQ9WoUUM9e/bUn3/+KX9//0xfv558rKdPn9bEiRPVuHFj+fj4qGPHjvr999+VlJSk2bNny8/PT9WrV1eHDh3u27f9+/erb9++aty4sapWrao6dero9ddfT/V+zOhxxMXFadasWXrhhRdUrVo1NWjQQP3799e5c+dStbt27Vp17NhRderUUY0aNdS+fXstXrw4XdeU+/n5qVOnTjp27Ji6desmHx8f+fr6avTo0YqIiEhVPzw8XCNHjpSvr6+qVq0qPz8/TZo0SdHR0SnqJb/G27Ztk5+fn3x8fPTee++l2Y8NGzYoKipKgYGBKl26dJr1PvroI40cOVLPPvvsA4/rwoULGjFihJo3b65q1aqpRo0aateunZYsWZKqbnrP36OcZwDILKZAA0AGOTk5acyYMeratatGjRqlOnXqyNnZ2ab7+PHHHxUcHKxWrVqpdu3aWr16tSZMmKALFy5o6dKl8vX1VadOnRQWFqbp06erYMGCevXVV1O0MWHCBMXHx6t169a6efOmwsLC9Prrrys4OFhNmza11vvyyy8VHBwsd3d3BQUFyWKxWOtOmDBBbdq0SdHu119/rfz586tr1646efKkKleunOZxhISEaMyYMXJxcZG/v7/y58+vHTt2aPTo0dq3b5+mTp2qggULqm/fvtq4caOOHTumV155RW5ubnJ3d0+z3R07dkiSGjdu/MDzeG/fk5KS9MEHHygsLEylS5dW+/btFRMTo02bNumNN97Q8OHD1aVLlxTbbNmyxXrO6tWrp127dum7777T+fPnNW/ePOs1uStWrNCFCxfUs2dP5cmTRwULFnxg3+61b98+vf322ypcuLBeeOEF5cmTR7t379bUqVN15swZjRs37oHb37x5U507d1a+fPnUtm1bXb16VevWrdMbb7yhdevWqUyZMpKkM2fOqGPHjoqIiFDz5s1VqlQpbdmyRZ07d1ZSUpJKlCiRoX7f6/3339eNGzcUEBCgS5cuacOGDXrzzTfl5+enrVu3qmXLloqNjdXq1av19ttvKywsTMWLF5ckbdy4Uf369VORIkXUvHlzFShQQCdOnND27dut195XqlQpw8cRHx+vnj17as+ePfL29lbXrl117do1rV+/Xjt37lRISIg8PDwkSevWrVP//v31zDPPKCgoSHZ2dtq0aZNGjRql69evq0+fPg89B1euXFH37t3l7u6uLl266NChQ1q0aJF+/vlnLVu2TPnz55ckXbx4UZ06ddLly5fVrFkzVahQQUePHtWcOXO0e/duLVq0yFpXkq5fv673339fzz//vJydnVON6t4teUT3ueeee2Bfa9euneqHmHudP39eHTp00K1bt+Tv76+SJUvq8uXL2rBhg0aOHKnExER17do1Q+fPFucZADLFAACky1dffWV4eHgYP/30k2EYhjF8+HDDw8PD+PTTT611bty4YXh4eBhdu3a1li1fvtzw8PAw5s6dm6rNrl27Gh4eHsaNGzcMwzCMc+fOGR4eHin2YxiGsWPHDmv5woULreXJ9Tt06JCqn3Xq1DHOnTtnLT98+LDh4+NjNG3a1EhISDAMwzAOHTpkeHp6Gl27djViYmKsdf/55x/D39/f8PHxMa5du2YYhmHs2bPH8PDwMHx8fIwrV6489HydPXvWqFy5stG0aVPj7Nmz1vKbN28a3bt3Nzw8PIwVK1ZYywcNGmR4eHgYR44ceWC7t27dMjw8PIyaNWs+tA/3WrFiheHh4WH06NHDuHnzZoq+NmrUyKhcubK1r3e/FuvWrbPWjYuLMwICAgwPDw/jzJkz1vJ7X0vDSP2euVuzZs2MWrVqWR+/++67hoeHR4pzFRcXZ7Rp08aoVKmSERkZaRjG/16HMWPGpNp37969jbi4OGv5jBkzDA8PD+OLL76wlvXq1cvw8PAw1q9fby2LjY01OnXqlOq9m5bk41q+fHmqsmbNmqU4Bx9++KH19fr777+t5dOmTTM8PDyMRYsWWctatmxp1K1b1wgPD0+xv2+++cbw8PAwpkyZkqnjmD17tuHh4WFMnDjRSEpKspb/9ttvRpUqVYz27dtby4KCgozq1asbUVFR1rKoqCijUaNGRv369VNsfz/NmjUzPDw8jLffftv6OTMMw/j0008NDw8P46uvvrKW9ezZ0/D09DQ2b96coo358+cbHh4exmeffWYtS36Nx48f/8D9J+vQoYPh4eFhHD58OF31kyW/73v37m0tS/5bt2vXrhR1Dx06ZHh4eBivvPKKtSy95+9RzzMAZBZToAEgkwYMGCA3NzctWrRIBw8etGnb7u7uat68ufVxzZo1JUn58+dXx44dreWlSpXSU089pQsXLqRqo3v37ipVqpT1ceXKlfXiiy/q4sWL2rdvn6Q7U4kNw9DAgQOVL18+a93ChQurZ8+eunXrltavX5+i3Vq1asnNze2hx7B69WolJCSoT58+KaZg5s+fX8OGDZMkLV++/KHt3Ct5uniBAgUyvG3yNOuRI0emGFkrXbq0evfurYSEBK1cuTLFNqVLl9a//vUv62NHR0c1aNBA0p3by9hK8pTwu6eUOzo6avbs2dq7d69cXFwe2kaPHj1STPdu0qRJin7+888/2rZtm2rXrq1WrVpZ6zk5Oemjjz6yxWGoXbt2KUa/k9+7AQEB1pFeSfL29pYk63s3KSlJ/fv318SJE1MtLlevXj1J0rVr1zJ1HKGhoXJxcdH7778vi8ViLa9WrZpatWql33//XSdOnJAkGYah27dvWxc1k+5cUx4aGqpNmzal2D4tFotFAwcOlL29vbXsvffeU/78+bVmzRpJd0aJt2/friZNmqhZs2Yptu/atatKlix539tMtWzZ8qH7lx7tc3KvF198UWPHjk21OJ23t7fy5s1rfV2k9J8/W5xnAMgMpkADQCa5uLho+PDh6tevn4YPH27Te6Leu4pwclgrUaJEii/VkpQnTx7rl927JQePu3l7e2vp0qU6duyY6tWrp8OHD0u6M+V669atKer+/fffkqSjR4+mKH/Q1OS7HTt2TNKda2LvVbFiRRUsWNBaJyOSF7y63zGnp0/Fixe/7zWRtWrVsta52zPPPJOqbnIYvfva3Ef10ksvaePGjRo0aJBmzJih5557Tr6+vqpfv36a1znf696+Jk/NT+7n4cOHlZSUZA2fd/Px8ZGDw6N/LUieap0s+YeVu3+Mke68b+/um52dnfz9/SXdCcUnTpzQ2bNndfLkSeu1wsk/EmTkOG7evKlTp07Jzc1NM2fOTFX/6tWrku68zytWrKhXXnlFI0aMsF6f7+vrqyZNmqhWrVrpvp2Wm5tbqttgubi4qFy5cjp8+LBu3bqlI0eOyDAMRUREaNq0aanacHR01KVLl3T58uUUPxyk9/NXuHBhnT59Wjdu3EhX/QdJniYdERGho0eP6uzZszp16pQOHjyo2NhYJSYmWuum9/zZ4jwDQGYQgAHgEbRs2VLPP/+8Nm3apDlz5qS6fjSz7h6NvVt6g5Ck+95HN3k0KCYmRpIUFRUlSfrmm2/SbOfeL9DJweVhkhfxSWvkslixYjpz5ky62rqbk5OTihcvrsuXL+vKlSsqVqxYmnWvXbsme3t7a2iOjo5O89ZVye3cvn071f7ulRWjU02aNNGCBQv07bffavfu3QoJCVFISIhcXV3Vt2/fdC1OdW9f7x5tk2RdDft+58De3t4m94K9e2T9QX27n+PHj2vMmDHWBcUcHR1VoUIFVa1aVadPn87UcSS/D8PDwzV9+vQ09538Pu/YsaOKFi2qBQsWaP/+/Tp+/Lhmz56t4sWLa/DgwXrhhRceehx3B9a7Jfc3KirK+gPOwYMHHziDJCIiIkV7efPmfej+pTs/OBw4cEBnz5697w8FyeLi4nT58uUHLpR148YNjR8/XmvXrlV8fLwsFovc3d1Vv379FCO4UvrPny3OMwBkBgEYAB7RiBEjtHfvXgUHB6tRo0apnn9QWLp161aW9Ss53N7typUrkqRChQpJuhNW7O3tdejQoRRTZ20hOWxfuXLlvsHqxo0bqW5flF7PPfecQkNDtWvXLgUFBaVZb/r06frPf/6jUaNG6eWXX1aBAgWs5+B+/ZGU6T7dz70B9G73e+3r1q2runXrKiYmRvv27dPWrVu1YsUKjRkzRmXKlLFOac6s5BHhe1cYTnbz5s1Hav9RREdHq0ePHoqKitKgQYPUsGFDlS9fXk5OTjp06JDWrl1rrZuR40gO5LVr19aiRYvS1Rd/f3/5+/srMjJSe/fu1ebNm7VmzRr1799fzz77rHXBrLTExsbetzw59Lq6ulr79c477zxwNefMeu6557RmzRrt2rVLrVu3TrPepk2b9P777+vFF1/UpEmT7ltnwIAB2rZtmzp27Kg2bdrIw8PD+hokT+m+W3rP36OeZwDIDOaYAMAjKl68uD788EPFxcVpxIgRqZ5PDpb3hgvDMO57CxZb+f3331OVJY80Va1aVZLk6empxMTEVNOcJenAgQOaPHmy9XrhjPLy8pKk+25/5swZhYeHq2LFiplqOzn0zp49+763gJJkXQXZYrFYr9n18vJSZGSk/vvf/6aqn9zPh90OJiPSeu0jIyNT3RJn/vz5+uKLLyTdCW2+vr765JNPrO+ph91uKj2qVKkii8Wi3377LdVzJ0+ezNEAvGfPHl29elVdunRRjx495OXlZR01/vPPPyX974eEjByHi4uLnn76aZ08eTLV6L4krVy5UtOmTdP58+cVFxenGTNmWG9ZVrBgQfn7+2v8+PHq3bu3kpKSdODAgYcey6lTp1L9AHXr1i0dP35clStXlpOTkzw9PSVJf/zxx33b+Oqrr/TNN99kepq9n5+fXF1dtWbNmjT/ziQlJWnhwoWSdN8f76Q779Vt27apatWqGjVqlGrWrGkNv+fPn1dsbKz1dUnv+bPVeQaAzCAAA4ANdO7cWTVq1Eg1HVCSypcvL+nOrXvuvlZu8eLF970vqK2EhIRY71Ur3Ql4YWFhqlixonVKZHKQHDduXIrRtOjoaI0cOVKzZ89O0eeMaNOmjRwcHDRz5swUX8BjYmI0evRoa53MqF27tl544QX9+eefevfdd1OFjStXrqhv376KiIhQ586drdM727VrJ0kaO3asdRq4JJ07d05ff/21HB0dFRAQkKk+3U/ya3/v9dUzZ860Xs+abOfOnZo5c2aq6bDJi0Q9/fTTj9yf4sWLq1GjRtq9e7e2bdtmLY+Li0tz9C+7JE+tv3tBJenOrYKSpy4n/9iR0eMICgpSRESEJk+enOK8nzx5UqNHj9bcuXPl6uoqJycnrV27Vl9++WWq0JiR1yE+Pl5Tp061BkPDMDRlyhTFxMSoffv2ku4srlanTh1t375dYWFhKbZfuXKlvv76a+3YsSNDlz3cLXnRr/j4eL355pv666+/UjwfGxtrvR1Z5cqV0xwldnR0lJ2dnSIjI1OE8du3b+vTTz+1Hq+kdJ8/W51nAMgMpkADgA1YLBaNGTNGbdu2tX4ZTFa5cmVVqVJFBw4cUOfOnVWnTh3997//1f/93//Jx8dHhw4dypI+OTg4qE2bNnrhhRd07do1hYWFKW/evBo/fry1Tv369dWtWzeFhIQoICBATZo0kZOTkzZu3KhLly6pY8eO1hV4M6p06dIaNGiQxo4dq6CgIDVv3lz58+fX9u3bde7cOQUEBKht27aZPr5x48YpKipKmzdvtq6kW6JECV24cEHbt2/XzZs35e/vr4EDB1q3adOmjTZv3qwNGzboxRdflK+vr/U+wFFRURo+fHiqRZweRZMmTVSsWDGtX79eUVFR8vLy0oEDB3TixAl5eHjo0qVL1rrvvvuu9u7dq+7du6tVq1YqXry4Tp48qS1btqhChQp68cUXbdKnoUOH6pVXXlHv3r3VvHlzFS9eXLt27bL+WJJTCxDVqlVL7u7uWrVqla5fvy4vLy9dunRJmzZtUp48eWSxWFL8YJSR43jrrbes9/vdv3+/6tatq8jISIWFhenWrVuaNGmSdVTzww8/VJ8+fRQUFKRWrVqpUKFC+uOPP7Rnzx7VrVs3zZHSuzk6OmrVqlU6evSo9TN+4MAB1atXL8Uq7qNHj1aXLl303nvvydfXVxUrVtSpU6e0detWubq63ndGSUZ06tRJV65cUXBwsFq3bq3GjRurQoUKioiI0J49e3Tx4kWVL19ewcHBaS6Ali9fPvn7+2vDhg166aWX1KhRI8XExGjLli26evWqChUqpKioKCUlJcnOzi7d588W5xkAMoMRYACwkWeffVZvvfXWfZ+bNWuWgoKCdPr0aS1cuFAxMTGaP3++fHx8sqw/48aNU9OmTfX9999r06ZNatSokZYuXapq1aqlqDds2DBNnDhRJUuW1OrVq7VixQo99dRTGjdu3CN/Ae/evbtmz56tKlWq6Mcff9SKFSvk6uqqMWPGaMqUKY/Udr58+TRr1ix9+eWXqlmzpg4cOKD58+dr165d8vHx0eeff67p06enGEGzWCz64osvNGzYMBUoUEChoaHasmWLqlevrnnz5tlsEbNkTk5OCgkJkb+/vw4ePKglS5bI2dlZS5YsSbXokLe3txYuXKhGjRppz549mjt3ro4fP67u3btr0aJFaS4ulVHly5fXkiVL1KRJE+3evVvfffedypQpo/nz50tKewG2rJY/f37NnTtXLVq00OHDh7Vw4UIdOXJEL774olavXi0vLy/t27fPOr05I8eRN29eLViwQO+++65iY2O1ePFibdu2TTVr1tSCBQsUGBhorfv888/r22+/VdWqVbVlyxYtWLBAf//9t/r06aNvvvkmXT8Q5M2bV4sXL5YkLVmyROHh4erbt6/mzJmTImiWL19e33//vV5++WUdP35cCxYs0PHjx9WmTRuFhobaZDr+e++9p6VLlyogIEBnz57Vf/7zH61fv16FCxfWRx99pBUrVqhkyZIPbGPcuHF69dVXFRUVpYULF2rHjh2qVq2alixZorZt2+r27dvWlbrTe/5scZ4BIDMsxv1W5gAAAE+cpKQknTt3Tk8//XSqRc/OnTun5s2bq1OnTho5cmTOdDCdHufj8PPzU2RkZKavnQcAZC1+XgMAwCQsFovatm2rwMDAVIsrffvtt5KU6Snv2elJOQ4AQPbjGmAAAEzCYrGoY8eO+ve//229Btre3l6//vqrDh48qMaNG6tVq1Y53c2HelKOAwCQ/ZgCDQCAiSQlJWn58uX67rvvdOrUKSUkJKhUqVIKDAzU66+/bvP7QWeVx/U4mAINAI83AjAAAAAAwBS4BhgAAAAAYAoEYAAAAACAKZhyEazr128qKSl3z/wuWtRZ165F53Q3gBzF5wBmx2cAZsdnAGbHZyA1OzuLChcukObzpgzASUlGrg/Akp6IYwAeFZ8DmB2fAZgdnwGYHZ+BjGEKNAAAAADAFAjAAAAAAABTIAADAAAAAEyBAAwAAAAAMAUCMAAAAADAFEy5CjQAAACA3OnWrZuKjo5QYmJCTnclx125YqekpKSc7ka2sbd3kLOzq/LlS/s2Rw9DAAYAAACQK9y6dVNRUdfl6uomR0cnWSyWnO5SjnJwsFNCgjkCsGEYio+PU0REuCRlOgQzBRoAAABArhAdHSFXVzc5OeUxffg1G4vFIienPHJ1dVN0dESm2yEAAwAAAMgVEhMT5OjolNPdQA5ydHR6pOnvBGAAAAAAuQYjv+b2qK8/ARgAAAAAYAoEYAAAAACAKbAKNAAAAIBcz6VgPuXNk3Px5nZsgqIib2V6+z17dmvZsiU6duyIYmNvq2RJd/n5NddLL3WSi4uLDXuaOb/+uk/9+r2tOXMWyMurssaOHaljx44oJGRZTnctQwjAAAAAAHK9vHkcFNh/VY7tf82UNorK5LYzZ07XwoXz1KxZcw0aNEzOzs46evSwli1boh9/XK8pU6bJ3b2UTftrVgRgAAAAAMghmzdv1MKF89Sv34d6+eXO1vJateqoefNW6t27h0aOHKqZM/8te3v7HOzpk4FrgAEAAAAgh4SE/FsVKjybIvwmK1GihHr27K2jRw9rzZoVaty4tn76KSxFnV27dqhx49q6cOG8JOn8+XMaMqS//P191apVU3366XBFRPzvvrljx47UkCH9NXLkULVo0UQjRgyRJJ09e1rDhg1S69bN1bRpfXXoEKh58+bIMIwsPPrsRwAGAAAAgBxw/fp1nTjxX9Wr1zDNOr6+zWSxWHTq1F+qWtVbW7ZsSvH85s0/qVKlKnJ3L6V//rmmd955U3//fUnDho3SRx8N0R9//K4PP+yj+Ph46za7d+9UXFycxo2bpDZt2ismJkbvvttLkZE3NHToSE2a9IVq1qytOXNmateuHVl2/DmBKdAAAAAAkAP+/vuiJKlkyafTrOPs7KyCBQvq778vyd+/pb7++ivFxMQof/78io+P165d29Wjx1uSpGXLliguLlaffx4sV1dXSVLlylXVqVM7bdy4Qf/6V2tJUmJiogYOHGqtc+zYUbm7l9aoUeNVuHBhSVKtWnW1fftWHTz4qxo39s2yc5DdGAEGAAAAgByQPL34Ydf22tvfGbf08/NXYmKCdu++Myr7f/+3WzExMfLzayHpzkrNVap4y9nZWQkJCUpISFCxYsX1zDPltH//L9b2XF0LW8OvJHl5VVJw8By5uLjo1Km/tH37Vs2dO1uJiQmKj4+z6THnNEaAAQAAACAHlChRUpJ0+fLfada5deuWbtyIULFiJVS4cBHVqlVXW7ZsVPPmLbVp04+qUaOWnnrqKUlSZOQNHTnyh5o2rZ+qnSJFilr/P3mU924LFvxbixcvUHR0tEqUKKmqVb1lb+/wxF0DTAAGAABPBFveA/RR7+eJR5fe19PN7eH3R+X1xOOqSJGiqlSpsnbs2Ko333xbFoslVZ1du7YrMTFRDRs2kiT5+7fU5MnjFRkZqZ07d6hv3/etdQsUcFb9+g315ptvp2onf/78afZj/fq1mj17hvr3H6TmzVvJ2dlZktS6tf+jHuJjhwD8hHucvwzYsm+xcYnK42SbZeHN8o+krW8Wb5bzBuDxZct7gD7K/Tyz2uP8b7stmeX1BF599U0NHvyhFi6cr27dXkvx3LVrVzVz5nR5eHipTp07o7pNmjTTpEnj9c03wYqLi1WTJn7W+t7e1bVz53aVL/+snJycJEmxsbEaPnyQfH2bqkyZZ+7bh8OHf5ebWzG1bdvBWnb8+DFFRFzXEzYATAB+0j3O/3jYum+P63E+rmx9s/jlE1qn61f49Hicv5ABQE57nP9tB5BxjRv76vXXe2rWrOn673+PqUWLVnJ2dtGJE8e1eHGIHB2dNPpPHjgAACAASURBVGrUODk43Ilu+fMXUMOGjbVmzQo1bNhYBQsWtLb1yitdFBb2gz76qJ9eeqmjHBwc9J//LNIff/ymnj17p9kHL6/KWrlyuebOna3q1WvqzJlTmjt3tiwWi2Jjb2f5OchOBGCkW1x8os0CDp48To72pvhCZpaRF8DsbPlvHp91IHvcjk3QmiltcnT/mfXGG73k7V1d3323RJMmjdfNm9EqWdJdgYFt9fLLneXikvLvkb9/K23dukktWrRKUV6iRAkFB8/RjBlfafToT2SxWOTp6aUvvghWxYqeae7/hRcCdfbsGa1cuVyLFs1XiRJPq1Onbjpz5rR+++1gpo/rcUQAfgzZ8poXW7JlwJGUo3+ggMyy5ciLLUfNbXkZgMQXdsCW/+bZ8rMO5Ea2/PH4yhU72dlZlJSUel5uVOStx/YH9PSoU6ee6tSpZ31sZ2eRnV3qa4Il6fnnn9fzz/963+cqVnxWU6d+laIsKcmwnrOhQ0em2sbOzk7vvNNP77zTL83+1axZWzt37rM+vl87uQEB+DFk66lNAB5Pth41t/UPVLn5S0ROYHYA0sIPyDmPEf2cZcvvth8Guevpp+8fgHPagwJrZp08F2GTdp4t7fpYnrOcQAAG8NjJyBeVh9XjiwqyC9dlAo8vs1ymg4yzdWi1VWCV7oRW2B4BGMBjhy8qTxZGRgEAjys7O4tNR1nx+CMAA7mIrW9dBGQHRkYBIPfhO8eTxTAkBwc7m7R19/XEuRHvaiAX4fpwwBxseRmAxMg5gIzjO8eTxWLheuJkBGAAAB4zWbFoEiPnAAAQgIFUbH2/Y1vfngYAAABA5hCAgXtkxcgLU4gAAACAnEcABgCkYuuZEAAAAI8DAjAAIBVb34oKTw5uawUAyM0IwAAAIN24rRWAx1XhQk5ycMqT4e3qFC5gk/0nxsU+0vb7f9mljRtW6cypk4qLi1WZMmXU4LmWata8tRwcbBvbFs4L1vYt65VkGBr48QR5eFV9pPZ+/XWf+vV7W3PmLJCXV2Ub9TJrEIABAAAA5HoOTnn019j2Obb/8kOXZ3rbeXO+1KYf16hxE3893+JF5c2bT5fPH9fCBTN15I8DeveDT2Rnb5tFVc+d+UthP4SqVUB71a7bWGWeqfDIbXp6emnmzLkqW7acDXqYtQjAAAAAAJBDdmz7URs3rFKPtz6Un39ra3nbgOdVoFBJTf/iU+3euUmNm7Swyf6io+/MvWnQ+HlVeNbLJm0WKOCsqlWr2aStrEYABgAAAIAc8sOqpSpTtnyK8JusfqNm+uvPY3J2KShJCr/yt5YsnKVjR35TXOxtVa5aQ527v60SJUtJkpYvm6eD+/foX4Ev6ftl83Xt6mWVKlNeQwYNVv7CZbV82Tyt+G6BJGnEkHfkVdlHvfoM0gd9OqvfhyNUt0ET674//qinyj7zrHr1HSRJ2r51g35Y9R+FX7mkQoVc1azZ8+rVq6/y5MmTagp0375vydPTS05OefTDD6t182a06tatr/79B+upp9wkSUlJSZo3b47WrFmpyMgbql+/oby9q2vatM+1c+e+LDvfBGAAAJ5wrOoNmIMtF6lD9rh+/ZrOnzulwLad0qzTuXtvSdK1a+EaMeQdFS7ylF578z3JMLQiNESjh7+nsRNnqXCRpyRJly6d1/Kl89T+5deUL38BLV30jYYOHajJ0xar6fMBKliosObP+VJvvTMw3SPAJ44f1uzgSerwymtq5ttAf/75p6ZN+1xOTnn09tt977vNDz+slqdnZQ0ZMlwRERH64otJ+uqrqRo9erwkafbsGVqyJEQ9erylihU9tXbtSs2a9XVGTl+m8AkBAOAJx6regDnYepE6ZL1/roVLkp5yK/7QumFrQxUXF6vBwyfJpWAhSVKlKtX1Qd8uWrfmO3V59U5Qvn0rRkOGT1KFipUkSUlJifp84nCdPf2nylXwkHupspKkUmXKyb30Mwq/8vdD93382O/KkzevXnjxZVUqX0ze3jXk6Ogge/u046Sdnb0mTvxcefLcWZjs5Mn/avXqlZKkmJibWrp0kbp1e13du/eQJNWv31CvvdZZf/554qH9eRQEYAAAAADIAXZ2dpKkJMN4aN1jR39TpSrVreFXklwKFlKVqjV07Mgha5m9vb3KVfC0Pi5S9M6U49jYzN92rqJHFcXevqWPP+qpf7Vqpfr1GykgoI0sFkua2zz7bEVr+JUkN7diun37Th8OH/5dcXFx8vVtan3eYrGoaVO/LA/AdlnaOgAAAADgvp566s7I77Xwy2nWuX79mpKSknQzOkqFXAuner6QaxHduhVjfezg4GgN1pJkZ0l/yE6LZ6Vq+mDgp3ItXFTz58/VW2+9pldeaavffjuY5jZ58+ZN8djOzk7G/+9DRESEJMn1nuMpXLhIpvuYXowAAwAAAEAOcClYSGXLPavfDv6ijl3fum+dCaM/UiHXInJ2dtGNiOupnr8R8Y+cnQtmug/Jg7j3BuTk0dpkNWs3VM3aDVWisL127typ+fO/1cCB72vNmp8yvM/khbAiIq7Lza2YtTziPsdna4wAAwAAAEAOaRXQQWfP/Kmtm9alem7n9p904fwZNXquuTy8quno4YOKirxhfT4q8oYO//6rPLyqZHr/+fIVkCRFXL9qLfvnWniKa4OXL5unER/3kSQ5O7uoefOW6ty5u6KjoxUdHZ3hfVas6Kn8+Qto587tKcp37dqexha2wwgwAACASdlyhfDbsQmKisz8NYaAWTX29dfB/Xv07ayp+vPEUdWs00h2dhatCf1NoaHfqV6DpvJt1kr/XAvXjq0bNOHTAWrboasMQ1q1fKEcHBzVMqBDpvdfwNlFFSpW0g+rl6lI0WKyt7fX98vmq4Czs7VO5So1tDI0RHNmTlb7toGKiLihBQv+LW/v6ipcuLBOncrYPp2dnfXyy50UEjJXjo6OqljRUxs2rNPx48ceeF2xLRCAAQAATMrWK4RH2aQlIHMS4mJVfujyHNt/YlxsprazWCzq8/4wVd70g7ZvCdMve3coISFeZcuWVfc33lVTvxdksVhU9KliGv7pF1oS8o1mTpsge3t7VapSXX0/GK6i/3+hq8zq1WeQ5s7+QjO+GqtCrkUUGNRZfxzab32+UhUf9XlvmNasXKIBAz6Qk1MeNWjQUH37fpDpfb7+ek8lJiZq2bIlunkzWg0bPqegoA5av/6HRzqWhyEAAwAAAMj1rt+IkxSXoW0cHOx08lyETfb/bGnXTG9rZ2en5/0D9bx/YIr27u1bqdLlNODj8Wm20/7l19T+5ddSlJUt96z27PnV2lblKtW18LvNKeo87V5GQ0dOTVF2d18kqX6jZqrfqJmeLe2qhISkFM/VrFlbO3fusz6ePv2bVH17+eXOevnlzpKk+Ph4bdr0o156qaN69epjrTNy5FCVKlUqzeOzBQIwAAAAACDbODo6at68OVqzZqU6d+6ufPny6Zdf9mrz5p80aNCwLN03ARgAAOQIW15/CgDIXSZO/EIzZkzT+PGjFBNzS2XKlNWQIZ/oX/9qnaX7JQADAIAcYcvrT6U716ACAHKHMmXKavz4ydm+X26DBAAAAAAwhWwfAZ4+fbrWr18vSWrSpIkGDhyoIUOGaP/+/cqXL58kqW/fvvL399fRo0c1dOhQ3bx5U7Vr19aoUaPk4OCgixcvasCAAbp27ZrKlSunyZMnq0CBAtl9KAAAAACAXCRbR4B3796tnTt3asWKFVq5cqUOHz6sn376SX/88YcWLlyoVatWadWqVfL395ckDRgwQJ988ok2bNggwzC0bNkySdKoUaPUuXNnhYWFqWrVqgoODs7OwwAAAACQAwxDMgwjp7uBHPSor3+2BmA3NzcNHjxYTk5OcnR0VIUKFXTx4kVdvHhRH3/8sQIDA/XVV18pKSlJFy5c0O3bt1W9enVJUrt27RQWFqb4+Hj98ssvatmyZYpyAAAAAE+26NuJisvk/XbxZIiPj5O9feYnMmdrAK5YsaI10J4+fVrr16/Xc889p/r162vcuHFatmyZ9u3bp9DQUF25ckVubv+7obObm5suX76s69evy9nZWQ4ODinKAQAAADzZNh+8roiIq4qLi2Uk2GQMw1BcXKwiIsLl7Jz5ey7nyCrQJ06cUK9evTRw4ECVL19eX3/9tfW5bt26aeXKlapQoYIsFou13DAMWSwW63/vdu/jhyla1PnRDgBArsJtVp4svJ7A48ssn0+zHOfj6M+/Y1WyZAldvnxF8fHxNmkzOjLGJu1cvHjDZm3Zuj1bt5VTHB0dVarU0ypUqFCm28j2ALx//37169dPH3/8sQICAnT8+HGdPn3aOqXZMAw5ODioRIkSCg8Pt2539epVFStWTEWKFFFUVJQSExNlb2+v8PBwFStWLEN9uHYtWklJj+8vRvxRBWwrPDzKZm3x+cx5tnw9bYn3BmCev7f8HcpZhQoVUlycbSayurm5aLCNbse2Zkobm7Vl6/Zs3VZOfgbi4h78GbSzszxwwDNbp0BfunRJffr00eTJkxUQECDpTuAdN26cbty4ofj4eC1dulT+/v5yd3dXnjx5tH//fknSqlWr5OvrK0dHR9WuXVvr1q2TJK1cuVK+vr7ZeRgAAAAAgFwoW0eAv/32W8XGxmrChAnWso4dO+qtt95Sp06dlJCQoBYtWqh169aSpMmTJ2vYsGGKjo5WlSpV1L17d0nSiBEjNHjwYM2YMUMlS5bU1KlTs/MwAAAAAAC5ULYG4GHDhmnYsGH3fa5Lly6pyry8vBQaGpqq3N3dXSEhITbvHwAAAADgyZWtU6ABAAAAAMgpBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCk45HQHAAAAgNwiLj5Rbm4uNmnrdmyCoiJv2aQtAOlDAAYAAADSycnRXoH9V9mkrTVT2ijKJi0BSC+mQAMAAAAATIERYAAAADwyW04NBoCsQgAGAADAI7Pl1GDpzvRgALA1pkADAAAAAEyBAAwAAAAAMAUCMAAAAADAFAjAAAAAAABTYBEsAIBpuRTMp7x5+KcQAACz4F99AIBp5c3jYLNVa1mxFgCAxx9ToAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgClkewCePn26AgICFBAQoIkTJ0qSdu/ercDAQLVo0UKff/65te7Ro0fVrl07tWzZUkOHDlVCQoIk6eLFi+rSpYtatWql3r176+bNm9l9GAAAAACAXCZbA/Du3bu1c+dOrVixQitXrtThw4e1du1affzxxwoODta6dev0xx9/aNu2bZKkAQMG6JNPPtGGDRtkGIaWLVsmSRo1apQ6d+6ssLAwVa1aVcHBwdl5GAAAAACAXChbA7Cbm5sGDx4sJycnOTo6qkKFCjp9+rTKli2r0qVLy8HBQYGBgQoLC9OFCxd0+/ZtVa9eXZLUrl07hYWFKT4+Xr/88otatmyZohwAAAAAgAfJ1gBcsWJFa6A9ffq01q9fL4vFIjc3N2udYsWK6fLly7py5UqKcjc3N12+fFnXr1+Xs7OzHBwcUpQDAAAAAPAgDjmx0xMnTqhXr14aOHCg7O3tdfr0aetzhmHIYrEoKSlJFoslVXnyf+927+OHKVrU+ZH6DyB3cXNzyekuwIZ4PQE8Sfibljmct5yVm89/tgfg/fv3q1+/fvr4448VEBCgn3/+WeHh4dbnw8PDVaxYMZUoUSJF+dWrV1WsWDEVKVJEUVFRSkxMlL29vbV+Rly7Fq2kJMNmx2RrufkNBTyOwsOjbNYWn8+cx+sJ4EnC37TMsdV5M9M5syVbvm9tzc7O8sABz2ydAn3p0iX16dNHkydPVkBAgCTJx8dHp06d0pkzZ5SYmKi1a9fK19dX7u7uypMnj/bv3y9JWrVqlXx9feXo6KjatWtr3bp1kqSVK1fK19c3Ow8DAAAAAJALZesI8LfffqvY2FhNmDDBWtaxY0dNmDBB7777rmJjY9WkSRO1atVKkjR58mQNGzZM0dHRqlKlirp37y5JGjFihAYPHqwZM2aoZMmSmjp1anYeBgAAAAAgF8rWADxs2DANGzbsvs+tXr06VZmXl5dCQ0NTlbu7uyskJMTm/QMAAAAAPLmydQo0AAAAAAA5hQAMAAAAADAFAjAAAAAAwBQIwAAAAAAAUyAAAwAAAABMgQAMAAAAADAFAjAAAAAAwBQIwAAAAAAAUyAAAwAAAABMgQAMAAAAADAFAjAAAAAAwBQIwAAAAAAAUyAAAwAAAABMgQAMAAAAADAFAjAAAAAAwBQIwAAAAAAAUyAAAwAAAABMgQAMAAAAADAFAjAAAAAAwBQIwAAAAAAAUyAAAwAAAABMgQAMAAAAADAFh5zuAABkpbj4RLm5ueR0NwAAAPAYIAADeKI5OdorsP8qm7W3Zkobm7UFAACA7MUUaAAAAACAKRCAAQAAAACmQAAGAAAAAJgCARgAAAAAYAoEYAAAAACAKRCAAQAAAACmQAAGAAAAAJgCARgAAAAAYAoEYAAAAACAKRCAAQAAAACmQAAGAAAAAJgCARgAAAAAYAoEYAAAAACAKRCAAQAAAACmQAAGAAAAAJgCARgAAAAAYAoEYAAAAACAKRCAAQAAAACm4JDTHQAAAACA9IqLT5Sbm0tOdwO5FAEYAAAAQK7h5GivwP6rbNLWmiltbNIOcg+mQAMAAAAATIEADAAAAAAwBQIwAAAAAMAUCMAAAAAAAFMgAAMAAAAATIFVoAEAAABkmaSEOJuutpyUEGeztmA+BGAAAAAAWcbOwUl/jW1vs/bKD11us7ZgPkyBBgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYAgEYAAAAAGAKBGAAAAAAgCkQgAEAAAAApkAABgAAAACYgkNOdwAAgPSKi0+Um5tLTncDAADkUgRgAECu4eRor8D+q2zW3popbWzWFgAAePwxBRoAAAAAYAoEYAAAAACAKRCAAQAAAACmkO0BODo6Wq1bt9b58+clSUOGDFGLFi3Upk0btWnTRj/99JMk6ejRo2rXrp1atmypoUOHKiEhQZJ08eJFdenSRa1atVLv3r118+bN7D4EAAAAAEAulK0B+NChQ+rUqZNOnz5tLfvjjz+0cOFCrVq1SqtWrZK/v78kacCAAfrkk0+0YcMGGYahZcuWSZJGjRqlzp07KywsTFWrVlVwcHB2HgIAAAAAIJfK1gC8bNkyjRgxQsWKFZMk3bp1SxcvXtTHH3+swMBAffXVV0pKStKFCxd0+/ZtVa9eXZLUrl07hYWFKT4+Xr/88otatmyZohwAAAAAgIfJ1tsgjR07NsXjq1evqn79+hoxYoRcXFzUq1cvhYaGqmLFinJzc7PWc3Nz0+XLl3X9+nU5OzvLwcEhRTkAAAAAAA+To/cBLl26tL7++mvr427dumnlypWqUKGCLBaLtdwwDFksFut/73bv4/QoWtQ5850GAAAAbMTNzSWnuwBkWG5+36Y7AI8ZMyZDDQ8bNuyhdY4fP67Tp09bpzQbhiEHBweVKFFC4eHh1npXr15VsWLFVKRIEUVFRSkxMVH29vYKDw+3TqfOiGvXopWUZGR4u+ySm99QAAAASL/w8CibtcV3SGQXW75vbc3OzvLAAc90B+A///xTR44c0Y0bN1SqVCkVK1ZMN27c0OnTp2UYhkqUKGGta7FY0hWADcPQuHHjVL9+feXPn19Lly5VUFCQ3N3dlSdPHu3fv1+1atXSqlWr5OvrK0dHR9WuXVvr1q1TYGCgVq5cKV9f3/QeAgAAAADAxNIdgFu0aKG//vpL33zzjXx8fKzlZ8+eVZ8+fRQUFKQePXpkaOdeXl5666231KlTJyUkJKhFixZq3bq1JGny5MkaNmyYoqOjVaVKFXXv3l2SNGLECA0ePFgzZsxQyZIlNXXq1AztEwAAAABgTukOwDNnztTAgQNThF9JKlOmjN577z2NGDEi3QF48+bN1v/v0qWLunTpkqqOl5eXQkNDU5W7u7srJCQkvd0GAAAAAEBSBm6DFB0dLTu7+1ePjo5WXFyczToFAAAAAICtpTsAN2rUSJMmTdKhQ4dSlO/du1dTpkxRq1atbN45AAAAAABsJd1ToIcPH67XXntNHTt2VMGCBVW4cGH9888/ioqKUt26dTVkyJCs7CcAAAAAAI8k3QHYzc1Nq1at0ubNm3Xw4EFFRUWpcOHCqlu3rho3bpyVfQQAAAAA4JGlOwBLkoODg1q0aKEqVaooPDxcFStWzKp+AQAAAABgUxkKwBs2bNCUKVN07tw5WSwWfffdd5o+fboKFCig8ePHy9HRMav6CQAAAADAI0n3Iljr1q3T+++/rzp16ujzzz9XUlKSJMnf318bN27U119/nWWdBAAAAADgUaV7BDg4OFjdu3fXkCFDlJiYaC1v166dIiMjFRISovfffz9LOgkAAAAAwKNK9wjwmTNn1KRJk/s+V6lSJYWHh9usUwAAAAAA2Fq6A/DTTz+t/fv33/e53377TSVLlrRZpwAAAAAAsLV0T4Hu0qWLJk6cKMMw1KRJE1ksFl2+fFlHjhzRzJkz9c4772RlPwEAAAAAeCTpDsDdu3dXZGSkZs+erRkzZsgwDL3zzjtycHBQt27d9MYbb2RlPwEAAAAAeCTpDsCxsbHq27evXn31VR08eFARERFycXGRt7e3ihQpkpV9BAAAAADgkaU7ALdt21YffPCBWrRooeeeey4r+wQAAAAAgM2lexGs69evq0CBAlnZFwAAAAAAsky6R4B79uyp8ePH67333lP58uVVtGjRVHVcXV1t2jkAAAAAAGwl3QF49uzZioqKUr9+/dKsc/ToUZt0CgAAAAAAW0t3AB40aFBW9gMAAAAAgCz1wADcuHFjzZ49W5UqVVJQUJAkKSIiQgULFpSdXbovHwYAAAAAIMc9MABfvXpV8fHx1seJiYlq0KCBQkNDVaVKlSzvnFklJcRpzZQ2NmsLAAAAAJCBKdDJDMPIin7gLnYOTvprbHubtFV+6HKbtAMAAAAAuR3zmAEAAAAApkAABgAAAACYQqYCsMVisXU/AAAAAADIUg+9Bvizzz6Ti4tLirJx48bJ2dk5RZnFYtGMGTNs2zsAAAAAAGzkgQG4Tp06kqSbN28+sAwAAAAAgMfdAwNwSEhIdvUDAAAAAIAsxSJYAAAAAABTIAADAAAAAEyBAAwAAAAAMAUCMAAAAADAFAjAAAAAAABTIAADAAAAAEzhgbdBAgAA2S8pIU5rprSxaXsAAIAADADAY8fOwUl/jW1vs/bKD11us7YAAMjNCMAAAAAA8ASz5cyi3D6riAAMAAD+X3v3G2RlWfh//LOwwPinvkTuCjIMY+W/TIXJUfEBWzYBshIMMqY0YtoDs0aTfmMqoI6OjOiQlJP4JFMzzVD5I4Sblelk4JA0I0OjliI0IsEqmULAAru/B477/SoZQgfOOXu9Xk/q3Ofs5XVm9p49b+7rvg4APVglVxa9u6poR0XGqgabYAEAAFAEAQwAAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwAAAARRDAAAAAFEEAAwAAUAQBDAAAQBEEMAAAAEUQwAAAABRBAAMAAFAEAQwAAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwAAAARRDAAAAAFEEAAwAAUAQBDAAAQBEEMAAAAEUQwAAAABRBAAMAAFAEAQwAAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwAAAARRDAAAAAFEEAAwAAUAQBDAAAQBEaqz0B6kfnro4s/v74io4HAABwsAhgPrJejX2zZua5FRvvU9MfrdhYAAAAe2MJNAAAAEUQwAAAABRBAAMAAFAEAQwAAEARDnoAb9myJeecc05ee+21JMmyZcsybty4jBo1KnPmzOl+3QsvvJCJEydm9OjRmT59enbt2pUkef311/O1r30tY8aMyWWXXZatW7ce7LcAAABAHTqoAfz888/nggsuyNq1a5Mk27dvz7Rp0zJ37twsXbo0q1evztNPP50kueqqq3L99dfnV7/6Vbq6ujJv3rwkyY033pjJkyenra0tn/vc5zJ37tyD+RYAAACoUwc1gOfNm5cbbrghzc3NSZJVq1Zl6NChGTJkSBobGzNu3Li0tbVl/fr12b59gfEmDwAAEzdJREFUe4YNG5YkmThxYtra2rJz58788Y9/zOjRo993HAAAAPbmoH4P8MyZM9/3eNOmTWlqaup+3NzcnI0bN+5xvKmpKRs3bsw//vGPHH744WlsbHzfcQAAANibgxrAH9TZ2ZmGhobux11dXWloaPjQ4+/97//1wccfxSc/efj+TxoAACqkqelj1Z4C7LN6/r2tagAPHDgw7e3t3Y/b29vT3Ny8x/E33ngjzc3NGTBgQN55553s3r07vXv37n79vnrzzS3p7OyqyHs4EOr5FwoAgI+uvf2dio3lMyQHSyV/byutV6+G/3jBs6pfg3TKKafk1Vdfzbp167J79+4sWbIkI0eOzODBg9OvX7+sXLkySbJo0aKMHDkyffr0yamnnpqlS5cmSRYuXJiRI0dW8y0AAABQJ6p6Bbhfv36ZNWtWLr/88uzYsSMtLS0ZM2ZMkmT27NmZMWNGtmzZkhNPPDFTpkxJktxwww255pprctddd2XQoEG5/fbbq/kWAAAAqBNVCeAnn3yy+/+PGDEijz322B6vOf744/PII4/scXzw4MG5//77D+j8AAAA6HmqugQaAAAADhYBDAAAQBEEMAAAAEUQwAAAABRBAAMAAFAEAQwAAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwAAAARRDAAAAAFEEAAwAAUITGak+AA6tzV0cWf398tacBAABQdQK4h+vV2DdrZp5bkbE+Nf3RiowDAABQDZZAAwAAUAQBDAAAQBEsgaZqKnl/cueujoqMAwAA9FwCmKpxfzIAAHAwWQINAABAEQQwAAAARRDAAAAAFEEAAwAAUAQBDAAAQBEEMAAAAEUQwAAAABRBAAMAAFAEAQwAAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwAAAARRDAAAAAFEEAAwAAUAQBDAAAQBEaqz0BAHq2zl0dWfz98RUbCwBgfwlgAA6oXo19s2bmuRUZ61PTH63IOABAmSyBBgAAoAgCGAAAgCIIYAAAAIoggAEAACiCTbAAAKAHsOs+7J0ABgCAHsCu+7B3AhgAgB7NlVHgPQIYAIAezZVR4D02wQIAAKAIrgADAADUmEou3ed/CWAAAIAaY+n+gWEJNAAAAEUQwAAAABTBEmgAAKBIviKrPAIYAAAokvtsy2MJNAAAAEUQwAAAABRBAAMAAFAEAQwAAEARbIIFVIRdFAGAklXysxAHjgAGKsIuigBAySr5WSjxeehAsQQaAACAIrgCDECxLN0HgLIIYACKZek+AJTFEmgAAACKIIABAAAoggAGAACgCAIYAACAItgEC4A9VHJ3ZACAWiGAAdiD3ZEBgJ5IAMMHVPrKl+8GBQCA2iCA4QMqeeUrcfULAPj3OnbuTlPTx6o9DSiKAAYAgCro26d3xv2/RRUbz94NsHd2gQYAAKAIAhgAAIAiWAINAFSFTQcBONgEMABQFaVsOij0AWqHAAYAOIBKCX2AeuAeYAAAAIpQM1eAL7zwwmzevDmNje9O6aabbsrWrVtzyy23ZMeOHTn77LMzderUJMkLL7yQ6dOnZ+vWrTn11FNz4403dv8cAAAA/Ds1UY1dXV1Zu3Ztfve733WH7Pbt2zNmzJjcf//9GTRoUC699NI8/fTTaWlpyVVXXZWbb745w4YNy7Rp0zJv3rxMnjy5yu8CAACAWlYTS6DXrFmTJLnkkkvyla98JT/72c+yatWqDB06NEOGDEljY2PGjRuXtra2rF+/Ptu3b8+wYcOSJBMnTkxbW1s1pw8AAEAdqIkAfvvttzNixIjceeeduffee/PQQw/l9ddfT1NTU/drmpubs3HjxmzatOl9x5uamrJx48ZqTBsAAIA6UhNLoIcPH57hw4d3P540aVLuuOOOfP7zn+8+1tXVlYaGhnR2dqahoWGP4/vik588/L+fNAAAQIGamj5W7Snst5oI4Oeeey47d+7MiBEjkrwbtYMHD057e3v3a9rb29Pc3JyBAwe+7/gbb7yR5ubmffrvvfnmlnR2dlVm8gdAPf9CAQAAPVt7+zvVnsKH6tWr4T9e8KyJJdDvvPNObrvttuzYsSNbtmzJggUL8t3vfjevvvpq1q1bl927d2fJkiUZOXJkBg8enH79+mXlypVJkkWLFmXkyJFVfgcAAADUupq4AvzFL34xzz//fCZMmJDOzs5Mnjw5w4cPz6xZs3L55Zdnx44daWlpyZgxY5Iks2fPzowZM7Jly5aceOKJmTJlSpXfAQAAALWuJgI4Sa688spceeWV7zs2YsSIPPbYY3u89vjjj88jjzxysKYGUPM6d3Vk8ffHV3saAAA1rWYCGDi4BFPP0quxb9bMPLdi431q+qMVGwsAoFYIYCiUYAIAoDQCGGAfVPLKeeeujoqMAwDARyOAAfZBJa+cu2pOPXL7BAD1TAADAB+ZfwQCoJ7VxPcAAwAAwIEmgAEAACiCJdAAVeJeSgCAg0sAA1SJeykBAA4uS6ABAAAoggAGAACgCJZAAwAA72OfCnoqAQwHWCX/gHTu6qjIOAAA/4l9KuipBDAcYP6AAABAbRDAANDDWcoIAO8SwADQw1mJwodxmw5QGgEMAPABpVw1948jQGkEMADABwhDgJ7J9wADAABQBAEMAABAEQQwAAAARRDAAAAAFMEmWABQAaXsGgwA9UwAAzXH91JSj+waDGXwNwrqmwAGao6QAKg/lV4FUatx6G8U1DcBDHXEEksAalUlwzARh8CBIYChjvhXZwAA2H92gQYAAKAIrgADANQRt8MA7D8BDABQR9wOA7D/LIEGAACgCAIYAACAIghgAAAAiiCAAQAAKIJNsACoG3a/BQD+GwKYHsGHYihDJXe/TeyACwClEcD0CL4SAgAA2BsBDABAzbG6CzgQBDAAADXH6i7gQBDAQI/mCgIAAO8RwECPZtMkAADe43uAAQAAKIIABgAAoAgCGAAAgCIIYAAAAIpgEywAAKBu+IYH/hsCGAAAqBu+I5r/hiXQAAAAFEEAAwAAUAQBDAAAQBEEMAAAAEWwCRYA0CPYGRaAvRHAAECPYGdYAPbGEmgAAACKIIABAAAoggAGAACgCAIYAACAIghgAAAAiiCAAQAAKIIABgAAoAgCGAAAgCIIYAAAAIoggAEAACiCAAYAAKAIAhgAAIAiCGAAAACKIIABAAAoggAGAACgCAIYAACAIghgAAAAiiCAAQAAKIIABgAAoAgCGAAAgCIIYAAAAIoggAEAAChCY7UnAAAAJerc1ZHF3x9f7WlAUQQwAABUQa/Gvlkz89yKjfep6Y9WbCzoqep2CfTixYszduzYjBo1Kg888EC1pwMAAECNq8srwBs3bsycOXMyf/789O3bN+eff35OP/30fOYzn6n21AAAAKhRdXkFeNmyZTnjjDPSv3//HHrooRk9enTa2tqqPS0AAABqWF1eAd60aVOampq6Hzc3N2fVqlUf+ed79Wo4ENOqqMb/adr7i+p8rEqPV6tjVXq8Wh2r0uPV6liVHq9Wx6r0eLU6VqXHq9WxKj1erY5V6fFqdaxKj1erY1V6vBLGqvR4tTpWpcer1bEqPV4lx6rlntrb3Bq6urq6DtJcKuauu+7Kjh07cuWVVyZJ5s2bl9WrV+emm26q8swAAACoVXW5BHrgwIFpb2/vftze3p7m5uYqzggAAIBaV5cBfOaZZ2b58uXZvHlztm3blieeeCIjR46s9rQAAACoYXV5D/CRRx6ZqVOnZsqUKdm5c2cmTZqUk08+udrTAgAAoIbV5T3AAAAAsK/qcgk0AAAA7CsBDAAAQBEEMAAAAEUQwAAAABRBAAMAAFAEAVyHFi9enLFjx2bUqFF54IEHqj0dOGi2bNmSc845J6+99lqSZNmyZRk3blxGjRqVOXPmVHl2cGD96Ec/Smtra1pbW3PbbbclcQ5Qnh/+8IcZO3ZsWltbc8899yRxHlCeW2+9Nddcc00Sv//7QwDXmY0bN2bOnDl58MEHs3DhwvziF7/Iyy+/XO1pwQH3/PPP54ILLsjatWuTJNu3b8+0adMyd+7cLF26NKtXr87TTz9d3UnCAbJs2bI888wzWbBgQRYuXJg///nPWbJkiXOAoqxYsSLPPvtsHnvssTz66KO5//778+KLLzoPKMry5cuzYMGCJD4L7S8BXGeWLVuWM844I/3798+hhx6a0aNHp62trdrTggNu3rx5ueGGG9Lc3JwkWbVqVYYOHZohQ4aksbEx48aNcy7QYzU1NeWaa65J375906dPn3z605/O2rVrnQMU5bTTTstPf/rTNDY25s0338zu3bvz9ttvOw8oxltvvZU5c+bkm9/8ZhKfhfaXAK4zmzZtSlNTU/fj5ubmbNy4sYozgoNj5syZOfXUU7sfOxcoyTHHHJNhw4YlSdauXZvHH388DQ0NzgGK06dPn9xxxx1pbW3NiBEj/C2gKNdff32mTp2aj3/840l8FtpfArjOdHZ2pqGhoftxV1fX+x5DKZwLlOivf/1rLrnkknzve9/LkCFDnAMU6Yorrsjy5cuzYcOGrF271nlAER5++OEMGjQoI0aM6D7ms9D+aaz2BNg3AwcOzHPPPdf9uL29vXtJKJRk4MCBaW9v737sXKCnW7lyZa644opMmzYtra2tWbFihXOAorzyyivp6OjICSeckEMOOSSjRo1KW1tbevfu3f0a5wE91dKlS9Pe3p7x48fnn//8Z/71r39l/fr1fv/3gyvAdebMM8/M8uXLs3nz5mzbti1PPPFERo4cWe1pwUF3yimn5NVXX826deuye/fuLFmyxLlAj7Vhw4Z8+9vfzuzZs9Pa2prEOUB5XnvttcyYMSMdHR3p6OjIb3/725x//vnOA4pwzz33ZMmSJVm0aFGuuOKKnHXWWfnxj3/s938/uAJcZ4488shMnTo1U6ZMyc6dOzNp0qScfPLJ1Z4WHHT9+vXLrFmzcvnll2fHjh1paWnJmDFjqj0tOCDuvvvu7NixI7Nmzeo+dv755zsHKEpLS0tWrVqVCRMmpHfv3hk1alRaW1szYMAA5wFF8llo/zR0dXV1VXsSAAAAcKBZAg0AAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwANSw3/zmN/nGN76RESNGZPjw4ZkwYUIeeOCB7Ny5M0kyf/78HHfccdm8eXOVZwoAtc/3AANAjbrxxhvz0EMPZcKECbngggty6KGHZsWKFbn11lvz7LPP5gc/+EG1pwgAdUUAA0ANWrhwYR588MHcdNNN+epXv9p9/Mwzz8yxxx6bqVOnZvHixVWcIQDUH0ugAaAG3X333TnuuOPeF7/vGTt2bC655JJ84hOf2OO5rq6u3HfffRk3blxOOumkDB8+PBdffHFeeuml7te0t7fnO9/5Tk4//fSccsopmTx5clasWPGRnweAeiWAAaDGbNq0KX/5y1/S0tLyoa+5+uqr/+3zP/nJTzJ79uxMmjQpd999d6677rq8/PLLufbaa7tfc+211+Zvf/tbbrnllsydOzeHHHJILr300rz11lsf6XkAqFeWQANAjfn73/+eJDnqqKP2+Wc3bNiQb33rW7nooouSJKeddlrefvvt3HLLLdm6dWsOO+ywPPfcc7nsssty1llnJUmOOeaY3HPPPdm2bVv69++/1+cBoF4JYACoMb17906SdHZ27vPPzpgxI0myefPmrFmzJmvWrMmTTz6ZJOno6Mhhhx2W4cOH54477shLL72UlpaWtLS05Oqrr+4eY2/PA0C9EsAAUGMGDRqU5N2ruR9m06ZNOeKII/Y4/sorr+S6667LypUrc8ghh+T444/PYYcdluTd+4OTZM6cObnzzjvz+OOP55e//GX69OmTiRMnZsaMGenbt+9enweAeuUeYACoMQMGDMhnP/vZ/P73v//Q11x88cW5+OKL33ess7Mzl112WXbt2pXFixfnT3/6Ux566KHupczv6d+/f6ZPn55nnnkmCxcuzIUXXph58+bl3nvv/UjPA0C9EsAAUIMuuuiivPjii3n44Yf3eG7RokV5+eWXM27cuPcd37x5c9atW5fzzjsvxx57bHr1evfP/P8N6c2bN+cLX/hCfv3rXydJTjjhhFx99dU56qijsmHDhr0+DwD1zBJoAKhB48ePz1NPPZXrr78+q1atype+9KU0NDTkmWeeyc9//vOcffbZOffcc7NgwYLunzniiCNy1FFH5b777ssRRxyRXr16ZeHChXnqqaeSJNu2bcvgwYMzdOjQ3Hzzzdm6dWsGDRqUp556KuvXr8+Xv/zlDBgw4D8+DwD1rKHrvRuCAICa0tnZmXnz5mX+/PlZt25dOjo6cvTRR+e8887LpEmT0tjYmPnz5+faa6/N8uXLM2DAgKxevTo333xzXnzxxRx++OE56aSTMmXKlHz961/P7bffntbW1rz55pu57bbb8oc//CFvvfVWjj766Fx66aU555xzkmSvzwNAvRLAAAAAFME9wAAAABRBAAMAAFAEAQwAAEARBDAAAABFEMAAAAAUQQADAABQBAEMAABAEQQwAAAARRDAAAAAFOH/AyGZojDVq+QlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x648 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(16, 9))\n",
    "plt.title(\"Number of Confusing Images per Class\", fontsize=20)\n",
    "plt.hist(pd_train['category'], bins=np.arange(43)-0.5, label='Overall')\n",
    "plt.hist(pd_confusing_images['category'], bins=np.arange(43)-0.5, label='Confusing')\n",
    "plt.xlabel('Class', fontsize=16)\n",
    "plt.ylabel('Freq', fontsize=16)\n",
    "plt.legend(fontsize=16)\n",
    "plt.savefig('confusion_hist____mistake_ratio={}.png'.format(mistakes_ratio), \n",
    "            #transparent=True,\n",
    "            bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.15394906634279643, 0.015394906634279643, 0.007697453317139821)"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_mistakes/len(pd_train), num_mistakes*0.1/len(pd_train), num_mistakes*0.05/len(pd_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def find_confusing_images(LOSS_LIMIT=9):\n",
    "#     confusing_images = []\n",
    "#     for i in range(42):\n",
    "#         pd_cat_i, y_pred_i, losses = calc_category_losses(i)\n",
    "#         pd_cat_i['predicted_category'] = y_pred_i.argmax(axis=1)\n",
    "#         pd_cat_i['loss'] = losses\n",
    "#         \n",
    "#         pd_weird_i = pd_cat_i[pd_cat_i['loss'] > LOSS_LIMIT].copy().sort_values(by=['loss'], ascending=False)\n",
    "#         confusing_images.append(pd_weird_i)\n",
    "#     return pd.concat(confusing_images, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd_confusing_images = find_confusing_images(LOSS_LIMIT=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n = len(pd_confusing_images)\n",
    "# print(n)\n",
    "# \n",
    "# pd_confusing_images_raprap = pd_confusing_images.iloc[:n//3].copy()\n",
    "# pd_confusing_images_ralph  = pd_confusing_images.iloc[n//3:2*(n//3)].copy()\n",
    "# pd_confusing_images_franz  = pd_confusing_images.iloc[2*(n//3):].copy()\n",
    "# \n",
    "# pd_confusing_images_raprap['correct_category'] = -1\n",
    "# pd_confusing_images_ralph['correct_category']  = -1\n",
    "# pd_confusing_images_franz['correct_category']  = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd_confusing_images_raprap.to_csv('shopee-product-detection-dataset/confusing_images__raprap.csv', index=False)\n",
    "# pd_confusing_images_ralph.to_csv('shopee-product-detection-dataset/confusing_images__ralph.csv',   index=False)\n",
    "# pd_confusing_images_franz.to_csv('shopee-product-detection-dataset/confusing_images__franz.csv',   index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd_confusing_images_franz.tail(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>0004b03ad7eabfb3989727c461310a84.jpg</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6397</th>\n",
       "      <td>0004d34271d0a0253f8a95d0746bae05.jpg</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2181</th>\n",
       "      <td>000d4ad3d0a372fd6eea2fee6bf738ef.jpg</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2626</th>\n",
       "      <td>0012f125ad07123e69468ae1d1d59de4.jpg</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2433</th>\n",
       "      <td>0015831b249b856e569e883a9dc6d4fa.jpg</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  filename  category\n",
       "485   0004b03ad7eabfb3989727c461310a84.jpg        43\n",
       "6397  0004d34271d0a0253f8a95d0746bae05.jpg        43\n",
       "2181  000d4ad3d0a372fd6eea2fee6bf738ef.jpg        43\n",
       "2626  0012f125ad07123e69468ae1d1d59de4.jpg        43\n",
       "2433  0015831b249b856e569e883a9dc6d4fa.jpg        43"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_test = pd.read_csv('shopee-product-detection-dataset/test.csv').sort_values(by=['filename'])\n",
    "pd_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 12186 validated image filenames.\n"
     ]
    }
   ],
   "source": [
    "test_datagen = tfk_ImageDataGenerator(\n",
    "    preprocessing_function=tfk_preprocess_input\n",
    ")\n",
    "\n",
    "test_generator = test_datagen.flow_from_dataframe(\n",
    "    pd_test,\n",
    "    directory='shopee-product-detection-dataset/test/',\n",
    "    x_col='filename',\n",
    "    target_size=input_shape[:2],\n",
    "    color_mode='rgb',\n",
    "    batch_size=batch_size,\n",
    "    class_mode=None,\n",
    "    shuffle=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96/96 [==============================] - 144s 2s/step\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(test_generator, verbose=1, steps=test_generator.n//batch_size+1)\n",
    "y_pred = y_pred[:test_generator.n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>0004b03ad7eabfb3989727c461310a84.jpg</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6397</th>\n",
       "      <td>0004d34271d0a0253f8a95d0746bae05.jpg</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2181</th>\n",
       "      <td>000d4ad3d0a372fd6eea2fee6bf738ef.jpg</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2626</th>\n",
       "      <td>0012f125ad07123e69468ae1d1d59de4.jpg</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2433</th>\n",
       "      <td>0015831b249b856e569e883a9dc6d4fa.jpg</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  filename  category\n",
       "485   0004b03ad7eabfb3989727c461310a84.jpg        12\n",
       "6397  0004d34271d0a0253f8a95d0746bae05.jpg        16\n",
       "2181  000d4ad3d0a372fd6eea2fee6bf738ef.jpg        40\n",
       "2626  0012f125ad07123e69468ae1d1d59de4.jpg        22\n",
       "2433  0015831b249b856e569e883a9dc6d4fa.jpg        12"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_submission = pd_test.copy()\n",
    "pd_submission['category'] = y_pred.argmax(axis=1)\n",
    "pd_submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_submission.to_csv('submission_1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}