data_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If this is being run locally and not on the kaggle platform, please run `data_download.sh`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['10253', '10254', '10255']\n"
     ]
    }
   ],
   "source": [
    "from os import walk, listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "# this will only work after deleting the zip file.\n",
    "for (dirpath, dirnames, filenames) in walk('./data'):\n",
    "    patients = dirnames\n",
    "    break\n",
    "print (patients)\n",
    "    \n",
    "files = []\n",
    "for i in patients:\n",
    "    for j in ['/0/', '/1/']:\n",
    "        path = './data/' + i + j\n",
    "        these_files = [(path+f) for f in listdir(path) if isfile(join(path, f))]\n",
    "        files += these_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib.image import imread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "imgs_0 = []\n",
    "imgs_1 = []\n",
    "for i in files:\n",
    "    if int(i[-5]) == 0:\n",
    "        imgs_0.append(imread(i))\n",
    "    else:\n",
    "        imgs_1.append(imread(i))\n",
    "imgs_0 = np.array(imgs_0)\n",
    "imgs_1 = np.array(imgs_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1432, 50, 50, 3), (237, 50, 50, 3))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgs_0.shape, imgs_1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## On data selection\n",
    "This dataset is very unbalanced. Efforts made to:\n",
    "- idc vs. non-idc is a 50/50 split\n",
    "- train vs. test is a 65/35 split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 50/50 split of IDC vs Non-IDC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_idc_n = imgs_0.shape[0]\n",
    "n_idc_n = imgs_1.shape[0]\n",
    "non_idc_to_delete = imgs_0.shape[0] - imgs_1.shape[0]\n",
    "# not inspected I know, but the dataset shows more non-idc than idc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1195,)\n"
     ]
    }
   ],
   "source": [
    "print(np.random.choice(imgs_0.shape[0], non_idc_to_delete, replace = False).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(237, 50, 50, 3)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = np.random.choice(imgs_0.shape[0], non_idc_to_delete, replace = False)\n",
    "imgs_0 = np.delete(imgs_0, idx, axis=0)\n",
    "imgs_0.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 65/35 split of training and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle and split\n",
    "# shuffle\n",
    "np.random.shuffle(imgs_0)\n",
    "np.random.shuffle(imgs_1)\n",
    "\n",
    "# split\n",
    "split_idx = int(np.floor(0.65 * imgs_0.shape[0]))  # idc and non idc share the same size\n",
    "\n",
    "train_0 = imgs_0[:split_idx]\n",
    "train_1 = imgs_1[:split_idx]\n",
    "\n",
    "test_0 = imgs_0[split_idx:]\n",
    "test_1 = imgs_1[split_idx:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((154, 50, 50, 3), (83, 50, 50, 3), (154, 50, 50, 3), (83, 50, 50, 3))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(train_0.shape, test_0.shape, train_1.shape, test_1.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cool! We have the dataset prepared (do not read 'cleaned') and wrapped in numpy arrays. Now we only have to create one-hot encodings.\n",
    "\n",
    "### Formatting for `tensorflow`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_0_labels = np.zeros(train_0.shape[0])\n",
    "test_0_labels = np.zeros(test_0.shape[0])\n",
    "\n",
    "train_1_labels = np.ones(train_1.shape[0])\n",
    "test_1_labels = np.ones(test_1.shape[0])\n",
    "\n",
    "train = np.append(train_0, train_1, axis=0)\n",
    "train_labels = np.append(train_0_labels, train_1_labels, axis=0)\n",
    "\n",
    "test = np.append(test_0, test_1, axis=0)\n",
    "test_labels = np.append(test_0_labels, test_1_labels, axis=0)\n",
    "\n",
    "def shuffle(a, b):\n",
    "    permutation = np.random.permutation(a.shape[0])\n",
    "    return a[permutation], b[permutation]\n",
    "\n",
    "train, train_labels = shuffle(train, train_labels)\n",
    "test, test_labels = shuffle(test, test_labels)\n",
    "\n",
    "def get_one_hot(targets, nb_classes):\n",
    "    # https://stackoverflow.com/a/42874726\n",
    "    return np.eye(nb_classes)[np.array(targets.astype(np.int32)).reshape(-1)]\n",
    "\n",
    "train_labels = get_one_hot(train_labels, 2)\n",
    "test_labels = get_one_hot(test_labels, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('train.npy', train, allow_pickle=False, fix_imports=False)\n",
    "np.save('train_labels.npy', train_labels, allow_pickle=False, fix_imports=False)\n",
    "np.save('test.npy', test, allow_pickle=False, fix_imports=False)\n",
    "np.save('test_labels.npy', test_labels, allow_pickle=False, fix_imports=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}