Active_Learn/AL_Augment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ezHo8joem7x6"
   },
   "source": [
    "# Active Learning (Labeling Selection) using Augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "firhzasTl2mP"
   },
   "outputs": [],
   "source": [
    "BASE_PATH = '/content/'\n",
    "MODEL_FNAME = 'model_stage1'\n",
    "ANNOTATIONS_FNAME = 'annotations.txt'\n",
    "NUM_ANNOTATE = 52\n",
    "FROM_STAGE1 = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ByBTkybeJrbJ"
   },
   "outputs": [],
   "source": [
    "![ ! -f \"pip_installed\" ] && \\\n",
    "pip install -q tensorflow-datasets==4.4.0 tensorflow-addons && \\\n",
    "unzip -qq /content/drive/MyDrive/TeamSemiSuperCV/Wing/xray_reborn.zip -d /root/tensorflow_datasets && \\\n",
    "unzip -qq /content/drive/MyDrive/TeamSemiSuperCV/Wing/XRay_.zip -d /content && \\\n",
    "unzip -qq /content/drive/MyDrive/TeamSemiSuperCV/Active_Learn/$MODEL_FNAME\\.zip -d /content/$MODEL_FNAME && \\\n",
    "cp /content/drive/MyDrive/TeamSemiSuperCV/Active_Learn/preprocess.py /content && \\\n",
    "cp /content/drive/MyDrive/TeamSemiSuperCV/Active_Learn/Xray_Reborn.py /content && \\\n",
    "cp /content/drive/MyDrive/TeamSemiSuperCV/Active_Learn/valid.txt /content && \\\n",
    "cp /content/drive/MyDrive/TeamSemiSuperCV/Active_Learn/test.txt /content && \\\n",
    "cp /content/drive/MyDrive/TeamSemiSuperCV/Active_Learn/$ANNOTATIONS_FNAME /content && \\\n",
    "git clone --depth 1 https://github.com/TeamSemiSuperCV/semi-super.git /content/semi-super && \\\n",
    "touch pip_installed\n",
    "!ls -F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bLcBMPG1rcX-"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "import logging\n",
    "logging.disable(logging.WARNING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "a6b67b69"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_datasets as tfds\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from importlib import reload\n",
    "from pathlib import Path\n",
    "from scipy.stats import entropy\n",
    "from scipy.special import softmax\n",
    "\n",
    "from preprocess import dict2dict, IMG_SIZE\n",
    "import Xray_Reborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MX2dRp1FC2Ik"
   },
   "outputs": [],
   "source": [
    "model_path = Path(BASE_PATH + MODEL_FNAME)\n",
    "if (model_path / 'saved_model.pb').exists():\n",
    "  model_path = str(model_path)\n",
    "else:\n",
    "  model_path = str([p.parent for p in model_path.glob('*/*/assets') if p.is_dir()][0])\n",
    "model_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RaFccABYG91H"
   },
   "source": [
    "## Labeling Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0e26bb2e"
   },
   "outputs": [],
   "source": [
    "ds = tfds.load('xray_reborn')\n",
    "\n",
    "ds_train = ds['train']\n",
    "AUTO = tf.data.AUTOTUNE\n",
    "ds_train = ds_train.map(dict2dict, num_parallel_calls=AUTO)\n",
    "ds_train = ds_train.batch(64).prefetch(AUTO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4bb75c71"
   },
   "outputs": [],
   "source": [
    "model = tf.saved_model.load(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6da32daa"
   },
   "outputs": [],
   "source": [
    "logits = []\n",
    "labels = []\n",
    "fnames = []\n",
    "\n",
    "for batch in ds_train:\n",
    "    outputs = model(batch['image'], trainable=False)\n",
    "    # ['block_group1', 'block_group3', 'block_group2', 'initial_conv', 'final_avg_pool', 'block_group4',\n",
    "    #  'sup_head_input', 'proj_head_output', 'proj_head_input', 'initial_max_pool', 'logits_sup'])\n",
    "    logits.append(outputs['logits_sup'].numpy())\n",
    "    labels.append(batch['label'].numpy())\n",
    "    fnames.append(batch['fname'].numpy())\n",
    "\n",
    "logits = np.concatenate(logits)\n",
    "initial_preds = np.argmax(logits, axis=1)\n",
    "initial_probs = softmax(logits, axis=1)[:, 1]\n",
    "labels = np.concatenate(labels)\n",
    "fnames = np.concatenate(fnames)\n",
    "fnames = [bs.decode('utf-8') for bs in fnames]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wMUlaMpWO1Qh"
   },
   "outputs": [],
   "source": [
    "def get_preds_probs(ds):\n",
    "  logits = []\n",
    "  for batch in ds:\n",
    "      outputs = model(batch['image'], trainable=False)\n",
    "      logits.append(outputs['logits_sup'].numpy())\n",
    "  logits = np.concatenate(logits)\n",
    "  probs = softmax(logits, axis=1)[:, 1]\n",
    "  preds = np.argmax(logits, axis=1)\n",
    "  return preds, probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K9hn5cs2R4KX"
   },
   "outputs": [],
   "source": [
    "NUM_AUGMENTS = 6\n",
    "\n",
    "len_train = len(ds['train'])\n",
    "board_score = np.zeros((len_train, NUM_AUGMENTS))\n",
    "board_probs = np.zeros((len_train, NUM_AUGMENTS))\n",
    "\n",
    "def update_scores(aug_num, new_preds):\n",
    "  same = (initial_preds == new_preds)\n",
    "  print((same).sum() / len_train)\n",
    "  board_score[:, aug_num] = np.where(same, 1, 0)\n",
    "\n",
    "def update_probs(aug_num, new_probs):\n",
    "  mse = ((new_probs - initial_probs)**2).mean()\n",
    "  print(mse, (new_probs.sum() / initial_probs.sum()))\n",
    "  board_probs[:, aug_num] = new_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gRifkINqPn5U"
   },
   "outputs": [],
   "source": [
    "def do_augment(aug_num, aug_fn, ds=ds_train):\n",
    "  ds_aug = ds.map(aug_fn, num_parallel_calls=AUTO)\n",
    "  preds, probs = get_preds_probs(ds_aug)\n",
    "  update_scores(aug_num, preds)\n",
    "  update_probs(aug_num, probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "whJjKkBLScDN"
   },
   "outputs": [],
   "source": [
    "def aug_nocrop(d):\n",
    "  image = d['image']\n",
    "  image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n",
    "  image = tf.image.resize(image, IMG_SIZE[:2])\n",
    "  d['image'] = image\n",
    "  return d\n",
    "\n",
    "ds_nocrop = ds['train'].map(aug_nocrop, num_parallel_calls=AUTO).batch(64).prefetch(AUTO)\n",
    "do_augment(0, lambda x: x, ds=ds_nocrop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Q71qrwfQJf3m"
   },
   "outputs": [],
   "source": [
    "def aug_crop_left(d):\n",
    "  image = d['image']\n",
    "  image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n",
    "  d['image'] = tf.image.crop_and_resize(image,\n",
    "                                        boxes=tf.constant([[0.0, 0.0, 1.0, 0.9]]),\n",
    "                                        box_indices=tf.constant([0]),\n",
    "                                        crop_size=IMG_SIZE[:2])\n",
    "  return d\n",
    "\n",
    "ds_crop = ds['train'].batch(1).map(aug_crop_left, num_parallel_calls=AUTO)\n",
    "ds_crop = ds_crop.unbatch().batch(64).prefetch(AUTO)\n",
    "do_augment(1, lambda x: x, ds=ds_crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4RtUz_hdLzQL"
   },
   "outputs": [],
   "source": [
    "def aug_crop_right(d):\n",
    "  image = d['image']\n",
    "  image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n",
    "  d['image'] = tf.image.crop_and_resize(image,\n",
    "                                        boxes=tf.constant([[0.0, 0.1, 1.0, 1.0]]),\n",
    "                                        box_indices=tf.constant([0]),\n",
    "                                        crop_size=IMG_SIZE[:2])\n",
    "  return d\n",
    "\n",
    "ds_crop = ds['train'].batch(1).map(aug_crop_right, num_parallel_calls=AUTO)\n",
    "ds_crop = ds_crop.unbatch().batch(64).prefetch(AUTO)\n",
    "do_augment(2, lambda x: x, ds=ds_crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_yv_JbssMzJM"
   },
   "outputs": [],
   "source": [
    "def aug_crop_upper(d):\n",
    "  image = d['image']\n",
    "  image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n",
    "  d['image'] = tf.image.crop_and_resize(image,\n",
    "                                        boxes=tf.constant([[0.0, 0.0, 0.9, 1.0]]),\n",
    "                                        box_indices=tf.constant([0]),\n",
    "                                        crop_size=IMG_SIZE[:2])\n",
    "  return d\n",
    "\n",
    "ds_crop = ds['train'].batch(1).map(aug_crop_upper, num_parallel_calls=AUTO)\n",
    "ds_crop = ds_crop.unbatch().batch(64).prefetch(AUTO)\n",
    "do_augment(3, lambda x: x, ds=ds_crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8WY5n0ZiMy8s"
   },
   "outputs": [],
   "source": [
    "def aug_crop_lower(d):\n",
    "  image = d['image']\n",
    "  image = tf.image.convert_image_dtype(image, dtype=tf.float32)\n",
    "  d['image'] = tf.image.crop_and_resize(image,\n",
    "                                        boxes=tf.constant([[0.1, 0.0, 1.0, 1.0]]),\n",
    "                                        box_indices=tf.constant([0]),\n",
    "                                        crop_size=IMG_SIZE[:2])\n",
    "  return d\n",
    "\n",
    "ds_crop = ds['train'].batch(1).map(aug_crop_lower, num_parallel_calls=AUTO)\n",
    "ds_crop = ds_crop.unbatch().batch(64).prefetch(AUTO)\n",
    "do_augment(4, lambda x: x, ds=ds_crop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4BEYqMXgRVa_"
   },
   "outputs": [],
   "source": [
    "def aug_noise(d):\n",
    "  d['image'] = tf.add(d['image'], tf.random.normal(tf.shape(IMG_SIZE), 0, 0.015))\n",
    "  return d\n",
    "\n",
    "do_augment(5, aug_noise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9BkLokWuQEPW"
   },
   "outputs": [],
   "source": [
    "# def aug_horz_flip(d):\n",
    "#   d['image'] = tf.image.flip_left_right(d['image'])\n",
    "#   return d\n",
    "\n",
    "# do_augment(6, aug_horz_flip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PEXH66v8OAz6"
   },
   "outputs": [],
   "source": [
    "# def aug_vert_flip(d):\n",
    "#   d['image'] = tf.image.flip_up_down(d['image'])\n",
    "#   return d\n",
    "\n",
    "# do_augment(7, aug_vert_flip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oYfbvyDXrdxL"
   },
   "outputs": [],
   "source": [
    "# def aug_brightness_incr(d):\n",
    "#   d['image'] = tf.image.adjust_brightness(d['image'], +0.15)\n",
    "#   return d\n",
    "\n",
    "# do_augment(8, aug_brightness_incr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "la_qAIMWsEFN"
   },
   "outputs": [],
   "source": [
    "# def aug_brightness_decr(d):\n",
    "#   d['image'] = tf.image.adjust_brightness(d['image'], -0.15)\n",
    "#   return d\n",
    "\n",
    "# do_augment(9, aug_brightness_decr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-ZoJ3-Ae0U8Y"
   },
   "outputs": [],
   "source": [
    "# def aug_contrast_incr(d):\n",
    "#   d['image'] = tf.image.adjust_contrast(d['image'], 1.0 + 0.2)\n",
    "#   return d\n",
    "\n",
    "# do_augment(10, aug_contrast_incr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "STs0yNOX0wk2"
   },
   "outputs": [],
   "source": [
    "# def aug_contrast_decr(d):\n",
    "#   d['image'] = tf.image.adjust_contrast(d['image'], 1.0 - 0.2)\n",
    "#   return d\n",
    "\n",
    "# do_augment(11, aug_contrast_decr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lbWzQlysS1l3"
   },
   "outputs": [],
   "source": [
    "# def aug_gamma_incr(d):\n",
    "#   d['image'] = tf.image.adjust_gamma(d['image'], 1.1)\n",
    "#   return d\n",
    "\n",
    "# do_augment(12, aug_gamma_incr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "P4kWLCRbgKc4"
   },
   "outputs": [],
   "source": [
    "# def aug_gamma_decr(d):\n",
    "#   d['image'] = tf.image.adjust_gamma(d['image'], 0.9)\n",
    "#   return d\n",
    "\n",
    "# do_augment(13, aug_gamma_decr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lkCKKBFPlB6t"
   },
   "outputs": [],
   "source": [
    "final_score = board_score.sum(axis=1)\n",
    "final_probs = np.var(board_probs, axis=1)\n",
    "print(final_score.max(), final_score.min(), final_score.mean())\n",
    "print(final_probs.max(), final_probs.min(), final_probs.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BfDIqZ4_n7um"
   },
   "outputs": [],
   "source": [
    "for fs in range(0, NUM_AUGMENTS):\n",
    "  print(f'{fs}:{(final_score == np.full_like(final_score, fs)).sum()}', end=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yRVEhL_Qn7mS"
   },
   "outputs": [],
   "source": [
    "ent_lbl_fname = list(zip(final_probs, labels, fnames))\n",
    "ent_lbl_fname.sort(key=lambda x: x[0], reverse=True)\n",
    "# ent_lbl_fname = [x for i, x in enumerate(ent_lbl_fname) if (i % 6) == 0]  # skip adjacents\n",
    "num_ones = sum(l for _, l, _ in ent_lbl_fname[:NUM_ANNOTATE])\n",
    "print(f'{num_ones}')\n",
    "selected = [f.strip() for _, _, f in ent_lbl_fname]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "v5y6CJoRU96a"
   },
   "outputs": [],
   "source": [
    "if FROM_STAGE1:\n",
    "  # ds_train_1pc = ds['train_1pc']\n",
    "  # annotations = {d['fname'].numpy().decode('utf-8') for d in ds_train_1pc}\n",
    "  annotations = set() \n",
    "elif os.path.isfile(ANNOTATIONS_FNAME):\n",
    "  print(ANNOTATIONS_FNAME)\n",
    "  with open(ANNOTATIONS_FNAME) as f:\n",
    "    annotations = set()\n",
    "    for line in f:\n",
    "      fname = line.strip()\n",
    "      if fname: annotations.add(fname)\n",
    "else:\n",
    "  print(f'{ANNOTATIONS_FNAME} not found!')\n",
    "  \n",
    "len(annotations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "83db3872"
   },
   "outputs": [],
   "source": [
    "annotate_new = set(selected[:NUM_ANNOTATE])\n",
    "annotate_thresh = NUM_ANNOTATE\n",
    "target_annotations = len(annotations) + NUM_ANNOTATE\n",
    "annotate_new |= annotations\n",
    "\n",
    "while len(annotate_new) < target_annotations:\n",
    "  annotate_thresh += 1\n",
    "  annotate_new.add(selected[annotate_thresh])\n",
    "  print('!', end='')\n",
    "print()\n",
    "len(annotate_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WoMZjImHjmaM"
   },
   "outputs": [],
   "source": [
    "with open('annotations.txt', 'w') as f:\n",
    "    for fname in annotate_new:\n",
    "        print(fname, file=f)\n",
    "!wc -l annotations.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NrDFx4qPtg9o"
   },
   "source": [
    "## Generate New Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "TfCUNKQhz5Q4"
   },
   "outputs": [],
   "source": [
    "# reload(Xray_Reborn)\n",
    "!rm -rf {BASE_PATH + 'xray_reborn'}\n",
    "ds = tfds.load('XrayReborn', data_dir=BASE_PATH)  # will re-generate TFDS dataset\n",
    "len(ds['train_act']), len(ds['validation']), len(ds['test'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p0_mbJoV-ZP_"
   },
   "source": [
    "## Stage 2 Fine-Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gNfgCqPFOD3H"
   },
   "outputs": [],
   "source": [
    "wc_annotations = !wc -l annotations.txt\n",
    "len_annotations = int(wc_annotations[0].split()[0])\n",
    "len_annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FHaf6iQPEb0c"
   },
   "outputs": [],
   "source": [
    "class simclrCommand():\n",
    "  def __init__(self, params):\n",
    "    self.params = params\n",
    "\n",
    "  def compile_command(self):\n",
    "    simclr_command = ['python3 /content/semi-super/run.py']\n",
    "    for k,v in self.params.items():\n",
    "      simclr_command.append(f'--{k}={v}')\n",
    "    return (\" \").join(simclr_command)\n",
    "\n",
    "  def run_command(self):\n",
    "    !{self.compile_command()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UYAVgxM1HkFF"
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    # Dataset\n",
    "    'dataset': 'xray_reborn',\n",
    "\n",
    "    # Training Logistics\n",
    "    'train_mode': 'finetune', \n",
    "    'mode': 'train_then_eval',\n",
    "    'train_split': 'train_act',\n",
    "    'eval_split': 'validation', \n",
    "    'checkpoint_epochs': 20,\n",
    "    'save_only_last_ckpt': True,\n",
    "    'eval_per_loop': False,\n",
    "    'zero_init_logits_layer': False,\n",
    "    'use_tpu': False,\n",
    "\n",
    "    # Training Hyperparams\n",
    "    'warmup_epochs': 0,\n",
    "    'train_epochs': 60,\n",
    "    'fine_tune_after_block': -1,\n",
    "    'train_batch_size': 14,\n",
    "    'learning_rate': 0.001, \n",
    "    'learning_rate_scaling': 'sqrt',\n",
    "    'weight_decay': 0.0, \n",
    "    'temperature': 0.1,\n",
    "\n",
    "    # Architecture\n",
    "    'image_size': 224,   \n",
    "    'resnet_depth': 50,\n",
    "    'width_multiplier': 2,\n",
    "    'sk_ratio':0.0625,  \n",
    "\n",
    "    # Augmentations\n",
    "    'color_jitter_strength': 0.5,\n",
    "    'use_blur': False, \n",
    "    'area_range_min': 1.0,\n",
    "\n",
    "    # Static\n",
    "    'data_dir': '/content/',\n",
    "    }\n",
    "\n",
    "if FROM_STAGE1:\n",
    "    params['zero_init_logits_layer'] = True\n",
    "\n",
    "slimsk2 = simclrCommand(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7Q7kGSD1LFDd"
   },
   "outputs": [],
   "source": [
    "# 1st Fine-Tuning /w Validation Split Results\n",
    "def FT1(run, rerun=False):\n",
    "  model_ft_name = f'model_{len_annotations}-{run}'\n",
    "  global model_ft_path\n",
    "  model_ft_path = BASE_PATH + model_ft_name\n",
    "  if not rerun:\n",
    "    !rm -rf $model_ft_path\n",
    "    assert not os.path.isdir(model_ft_path)\n",
    "\n",
    "  slimsk2.params['mode'] = 'train_then_eval'\n",
    "  slimsk2.params['checkpoint'] = model_path\n",
    "  slimsk2.params['model_dir'] = model_ft_path\n",
    "  slimsk2.run_command()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "exTiaFF2LPbd"
   },
   "outputs": [],
   "source": [
    "# 2nd (Follow-on) Fine-Tuning /w Validation Split Results\n",
    "def FT2(run, rerun=False):\n",
    "  model_ft_name = f'model_{len_annotations}-{run}'\n",
    "  model_ft_path = BASE_PATH + model_ft_name\n",
    "  model_ft_path_sm = str([p.parent for p in Path(model_ft_path).glob('*/*/assets') if p.is_dir()][0])\n",
    "  global model_ft2_path\n",
    "  model_ft2_path = model_ft_path + '+'\n",
    "  if not rerun:\n",
    "    !rm -rf $model_ft2_path\n",
    "    assert not os.path.isdir(model_ft2_path)\n",
    "\n",
    "  slimsk2.params['mode'] = 'train_then_eval'\n",
    "  slimsk2.params['checkpoint'] = model_ft_path_sm\n",
    "  slimsk2.params['model_dir'] = model_ft2_path\n",
    "  slimsk2.run_command()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Ivad9UKFMfTy"
   },
   "outputs": [],
   "source": [
    "FT1(1) # Run #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nMXUFPg1MkNZ"
   },
   "outputs": [],
   "source": [
    "FT1(2) # Run #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ewtP3L62MqU7"
   },
   "outputs": [],
   "source": [
    "FT1(3) # Run #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "V2Bx7m5qMtg4"
   },
   "outputs": [],
   "source": [
    "FT1(4) # Run #4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XxWpB0ayciBE"
   },
   "outputs": [],
   "source": [
    "# Average Validation Accuracy based on 4 Runs\n",
    "avg_eval_accuracy = np.mean([0.956107, 0.956107, 0.948473, 0.948473])\n",
    "avg_eval_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zi8ARQleeSd1"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyN8i0BAIg1XYwmaa0dyhj9U",
   "mount_file_id": "1gR3JfM4sv_jwzOgX2PS4eN6-kouXHvw-",
   "name": "Augment.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}