00_fetching_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Genetic attribution preprocessing\n",
    "\n",
    "In this notebook we overview the [Genetic Engineering Attrinution competition in Driven Data](https://www.drivendata.org/competitions/63/genetic-engineering-attribution/).   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 279 ms, sys: 20.3 ms, total: 300 ms\n",
      "Wall time: 299 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import csv\n",
    "import requests\n",
    "from tqdm import tqdm\n",
    "from collections import OrderedDict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/rio/anaconda3/envs/genetic_attribution/bin/conda\r\n"
     ]
    }
   ],
   "source": [
    "!which conda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/rio/anaconda3/envs/genetic_attribution/bin/pip\r\n"
     ]
    }
   ],
   "source": [
    "!which pip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating tree structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_tree(generate_dirs,erase=False):\n",
    "    pbar = tqdm(generate_dirs)\n",
    "    for d in pbar:\n",
    "        if not os.path.isdir(d):\n",
    "            pbar.set_description(f\"Generating directory {d}\")\n",
    "            os.mkdir(d)\n",
    "        elif erase:\n",
    "            pbar.set_description(f\"Erasing directory {d}\")\n",
    "            pbar.set_description(f\"Generating directory {d}\")\n",
    "            os.mkdir(d)\n",
    "        else:\n",
    "            pbar.set_description(f\"Directory {d} already exists. Skipping.\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating directory /home/rio/data_sets/genetic_engineering_attribution/pca_engineered_datasets: 100%|██████████| 8/8 [00:00<00:00, 2692.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Current directory /home/rio/data_sets/genetic_engineering_attribution\n",
      "['val', 'string_patterns', 'pca_engineered_datasets', 'pca', 'test', 'train', 'original_data']\n",
      "CPU times: user 8.92 ms, sys: 79 µs, total: 9 ms\n",
      "Wall time: 7.22 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#parent_dir = \"/content/genetic_engineering_attribution\"\n",
    "parent_dir = \"/home/rio/data_sets/genetic_engineering_attribution\"\n",
    "\n",
    "### directory to put downloaded files\n",
    "original_data_dir = os.path.join(parent_dir,\"original_data\") \n",
    "\n",
    "### directory to put string patterns\n",
    "string_patterns_dir = os.path.join(parent_dir, \"string_patterns\")\n",
    "\n",
    "### directories to put train, val and test data sets\n",
    "train_dir = os.path.join(parent_dir,\"train\")\n",
    "val_dir = os.path.join(parent_dir,\"val\")\n",
    "test_dir = os.path.join(parent_dir,\"test\")\n",
    "\n",
    "### directory with fitted pca objects\n",
    "pca_dir = os.path.join(parent_dir,\"pca\")\n",
    "\n",
    "### directory with pca engineered data sets\n",
    "pca_engineered_data_dir = os.path.join(parent_dir,\"pca_engineered_datasets\")\n",
    "\n",
    "generate_dirs = [parent_dir, original_data_dir, string_patterns_dir, pca_dir, train_dir, val_dir, test_dir,\\\n",
    "                 pca_engineered_data_dir]\n",
    "            \n",
    "### generating tree\n",
    "erase=False\n",
    "generate_tree(generate_dirs,erase)\n",
    "\n",
    "###checking\n",
    "os.chdir(parent_dir)\n",
    "current_dir = os.getcwd()\n",
    "print(\"\\n\")\n",
    "print(f\"Current directory {current_dir}\")\n",
    "print(os.listdir(current_dir))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetching data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_data(url_dict,overwrite=False):\n",
    "    pbar = tqdm(url_dict.items(),total=len(url_dict))\n",
    "    for data,locs in pbar:\n",
    "        #print(\"value: \", value)\n",
    "        url, path = locs\n",
    "        if os.path.isfile(path):\n",
    "            if not overwrite:\n",
    "                pbar.set_description(f\"File {path} already exists. Skipping.\")\n",
    "                continue\n",
    "            else:\n",
    "                pbar.set_description(f\"File {path} already exists. Overwriting.\")\n",
    "        else:\n",
    "            pbar.set_description(f\"Writing {path}.\")\n",
    "        response = requests.get(url)   \n",
    "        with open(path, 'w') as f:\n",
    "            writer = csv.writer(f)\n",
    "            for line in response.iter_lines():\n",
    "                writer.writerow(line.decode('utf-8').split(','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Writing /home/rio/data_sets/genetic_engineering_attribution/original_data/test_values.csv.: 100%|██████████| 3/3 [14:05<00:00, 281.89s/it] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contents of directories: \n",
      "\n",
      "original_data_dir:  ['train_values.csv', 'test_values.csv', 'train_labels.csv']\n",
      "\n",
      "\n",
      "train_dir:  []\n",
      "\n",
      "\n",
      "val_dir:  []\n",
      "\n",
      "\n",
      "test_dir:  []\n",
      "\n",
      "\n",
      "CPU times: user 29.8 s, sys: 11.9 s, total: 41.7 s\n",
      "Wall time: 14min 5s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "####### training data\n",
    "# training features\n",
    "train_values_url = \"https://drivendata-prod.s3.amazonaws.com/data/63/public/train_values.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIARVBOBDCY3EFSLNZR%2F20200921%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200921T191351Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=f3b5911cbc8795264f6b8ffde9aee6f3b681f29bdbdf14c773120bf0192f86e7\"\n",
    "train_values_path = os.path.join(original_data_dir,\"train_values.csv\")\n",
    "# training targets\n",
    "train_labels_url = \"https://drivendata-prod.s3.amazonaws.com/data/63/public/train_labels.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIARVBOBDCY3EFSLNZR%2F20200921%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200921T191351Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=52fd0fd62b8627f6e530bc208058d322da578aa2b7b8f4f286c232ce00c6e34c\" \n",
    "train_labels_path = os.path.join(original_data_dir,\"train_labels.csv\")\n",
    "####### test data\n",
    "# test features\n",
    "test_values_url = \"https://drivendata-prod.s3.amazonaws.com/data/63/public/test_values.csv?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIARVBOBDCY3EFSLNZR%2F20200921%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200921T191351Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=d2b1adab04148d904aa40674c6c2605a81e9e20e640e7989e89d8aafd0ac776a\"\n",
    "test_values_path = os.path.join(original_data_dir,\"test_values.csv\")\n",
    "####### url_dict\n",
    "url_dict = {\"train_values\": (train_values_url, train_values_path), \"train_labels\": (train_labels_url, train_labels_path), \"test_values\": (test_values_url, test_values_path)} \n",
    "####### fetching_data\n",
    "overwrite = False\n",
    "fetch_data(url_dict,overwrite)\n",
    "######## Printing contents of directories\n",
    "print(\"Contents of directories: \\n\")\n",
    "print(\"original_data_dir: \", os.listdir(original_data_dir))\n",
    "print(\"\\n\")\n",
    "print(\"train_dir: \", os.listdir(train_dir))\n",
    "print(\"\\n\")\n",
    "print(\"val_dir: \", os.listdir(val_dir))\n",
    "print(\"\\n\")\n",
    "print(\"test_dir: \", os.listdir(test_dir))\n",
    "print(\"\\n\")\n",
    "#print(\"seqs_test_dir: \", os.listdir(seqs_test_dir))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading train_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of train_values:  (63017, 41)\n",
      "CPU times: user 1.64 s, sys: 65.9 ms, total: 1.7 s\n",
      "Wall time: 1.7 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>copy_number_low_copy</th>\n",
       "      <th>copy_number_unknown</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>CATGCATTAGTTATTAATAGTAATCAATTACGGGGTCATTAGTTCA...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5SAQC</td>\n",
       "      <td>GCTGGATGGTTTGGGACATGTGCAGCCCCGTCTCTGTATGGAGTGA...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E7QRO</td>\n",
       "      <td>NNCCGGGCTGTAGCTACACAGGGCGGAGATGAGAGCCCTACGAAAG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CT5FP</td>\n",
       "      <td>GCGGAGATGAAGAGCCCTACGAAAGCTGAGCCTGCGACTCCCGCAG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7PTD8</td>\n",
       "      <td>CGCGCATTACTTCACATGGTCCTCAAGGGTAACATGAAAGTGATCC...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id                                           sequence  \\\n",
       "0       9ZIMC  CATGCATTAGTTATTAATAGTAATCAATTACGGGGTCATTAGTTCA...   \n",
       "1       5SAQC  GCTGGATGGTTTGGGACATGTGCAGCCCCGTCTCTGTATGGAGTGA...   \n",
       "2       E7QRO  NNCCGGGCTGTAGCTACACAGGGCGGAGATGAGAGCCCTACGAAAG...   \n",
       "3       CT5FP  GCGGAGATGAAGAGCCCTACGAAAGCTGAGCCTGCGACTCCCGCAG...   \n",
       "4       7PTD8  CGCGCATTACTTCACATGGTCCTCAAGGGTAACATGAAAGTGATCC...   \n",
       "\n",
       "   bacterial_resistance_ampicillin  bacterial_resistance_chloramphenicol  \\\n",
       "0                              0.0                                   0.0   \n",
       "1                              1.0                                   0.0   \n",
       "2                              1.0                                   0.0   \n",
       "3                              1.0                                   0.0   \n",
       "4                              1.0                                   0.0   \n",
       "\n",
       "   bacterial_resistance_kanamycin  bacterial_resistance_other  \\\n",
       "0                             1.0                         0.0   \n",
       "1                             0.0                         0.0   \n",
       "2                             0.0                         0.0   \n",
       "3                             0.0                         0.0   \n",
       "4                             0.0                         0.0   \n",
       "\n",
       "   bacterial_resistance_spectinomycin  copy_number_high_copy  \\\n",
       "0                                 0.0                    1.0   \n",
       "1                                 0.0                    1.0   \n",
       "2                                 0.0                    1.0   \n",
       "3                                 0.0                    1.0   \n",
       "4                                 0.0                    1.0   \n",
       "\n",
       "   copy_number_low_copy  copy_number_unknown  ...  species_budding_yeast  \\\n",
       "0                   0.0                  0.0  ...                    0.0   \n",
       "1                   0.0                  0.0  ...                    0.0   \n",
       "2                   0.0                  0.0  ...                    0.0   \n",
       "3                   0.0                  0.0  ...                    0.0   \n",
       "4                   0.0                  0.0  ...                    0.0   \n",
       "\n",
       "   species_fly  species_human  species_mouse  species_mustard_weed  \\\n",
       "0          0.0            0.0            1.0                   0.0   \n",
       "1          0.0            0.0            1.0                   0.0   \n",
       "2          0.0            0.0            1.0                   0.0   \n",
       "3          0.0            0.0            1.0                   0.0   \n",
       "4          0.0            0.0            1.0                   0.0   \n",
       "\n",
       "   species_nematode  species_other  species_rat  species_synthetic  \\\n",
       "0               0.0            0.0          0.0                0.0   \n",
       "1               0.0            0.0          0.0                0.0   \n",
       "2               0.0            0.0          0.0                0.0   \n",
       "3               0.0            0.0          0.0                0.0   \n",
       "4               0.0            0.0          0.0                0.0   \n",
       "\n",
       "   species_zebrafish  \n",
       "0                0.0  \n",
       "1                0.0  \n",
       "2                0.0  \n",
       "3                0.0  \n",
       "4                0.0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_values = pd.read_csv(train_values_path)\n",
    "print(\"shape of train_values: \", train_values.shape)\n",
    "train_values.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of train_labels:  (63017, 1315)\n",
      "CPU times: user 5.1 s, sys: 115 ms, total: 5.22 s\n",
      "Wall time: 5.21 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>00Q4V31T</th>\n",
       "      <th>012VT4JK</th>\n",
       "      <th>028IO5W2</th>\n",
       "      <th>03GRNN7N</th>\n",
       "      <th>03Y3W51H</th>\n",
       "      <th>09MQV1TY</th>\n",
       "      <th>0A4AHRCT</th>\n",
       "      <th>0A9M05NC</th>\n",
       "      <th>0B9GCUVV</th>\n",
       "      <th>...</th>\n",
       "      <th>ZQNGGY33</th>\n",
       "      <th>ZSHS4VJZ</th>\n",
       "      <th>ZT1IP3T6</th>\n",
       "      <th>ZU6860XU</th>\n",
       "      <th>ZU6TVFFU</th>\n",
       "      <th>ZU75P59K</th>\n",
       "      <th>ZUI6TDWV</th>\n",
       "      <th>ZWFD8OHC</th>\n",
       "      <th>ZX06ZDZN</th>\n",
       "      <th>ZZJVE4HO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5SAQC</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E7QRO</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CT5FP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7PTD8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1315 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id  00Q4V31T  012VT4JK  028IO5W2  03GRNN7N  03Y3W51H  09MQV1TY  \\\n",
       "0       9ZIMC       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "1       5SAQC       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "2       E7QRO       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "3       CT5FP       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "4       7PTD8       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   0A4AHRCT  0A9M05NC  0B9GCUVV  ...  ZQNGGY33  ZSHS4VJZ  ZT1IP3T6  ZU6860XU  \\\n",
       "0       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "1       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "2       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "3       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "4       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   ZU6TVFFU  ZU75P59K  ZUI6TDWV  ZWFD8OHC  ZX06ZDZN  ZZJVE4HO  \n",
       "0       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "1       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "2       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "3       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "4       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "\n",
       "[5 rows x 1315 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "train_labels = pd.read_csv(train_labels_path)\n",
    "print(\"shape of train_labels: \", train_labels.shape)\n",
    "train_labels.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading test_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of test_labels:  (18816, 41)\n",
      "CPU times: user 485 ms, sys: 0 ns, total: 485 ms\n",
      "Wall time: 484 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>copy_number_low_copy</th>\n",
       "      <th>copy_number_unknown</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>E0VFT</td>\n",
       "      <td>AGATCTATACATTGAATCAATATTGGCAATTAGCCATATTAGTCAT...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TTRK5</td>\n",
       "      <td>GCGCGCGTTGACATTGATTATTGACTAGTTATTAATAGTAATCAAT...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2Z7FZ</td>\n",
       "      <td>GCTTAAGCGGTCGACGGATCGGGAGATCTCCCGATCCCCTATGGTG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VJI6E</td>\n",
       "      <td>ATGATGATGATGTCCCTGAACAGCAAGCAGGCGTTTAGCATGCCGC...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>721FI</td>\n",
       "      <td>GGTACCGAGCTCTTACGCGTGCTAGCCATACTATCAGCCACTTGTG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id                                           sequence  \\\n",
       "0       E0VFT  AGATCTATACATTGAATCAATATTGGCAATTAGCCATATTAGTCAT...   \n",
       "1       TTRK5  GCGCGCGTTGACATTGATTATTGACTAGTTATTAATAGTAATCAAT...   \n",
       "2       2Z7FZ  GCTTAAGCGGTCGACGGATCGGGAGATCTCCCGATCCCCTATGGTG...   \n",
       "3       VJI6E  ATGATGATGATGTCCCTGAACAGCAAGCAGGCGTTTAGCATGCCGC...   \n",
       "4       721FI  GGTACCGAGCTCTTACGCGTGCTAGCCATACTATCAGCCACTTGTG...   \n",
       "\n",
       "   bacterial_resistance_ampicillin  bacterial_resistance_chloramphenicol  \\\n",
       "0                              1.0                                   0.0   \n",
       "1                              0.0                                   0.0   \n",
       "2                              1.0                                   0.0   \n",
       "3                              0.0                                   0.0   \n",
       "4                              1.0                                   0.0   \n",
       "\n",
       "   bacterial_resistance_kanamycin  bacterial_resistance_other  \\\n",
       "0                             0.0                         0.0   \n",
       "1                             0.0                         1.0   \n",
       "2                             0.0                         0.0   \n",
       "3                             0.0                         0.0   \n",
       "4                             0.0                         0.0   \n",
       "\n",
       "   bacterial_resistance_spectinomycin  copy_number_high_copy  \\\n",
       "0                                 0.0                    0.0   \n",
       "1                                 0.0                    1.0   \n",
       "2                                 0.0                    0.0   \n",
       "3                                 1.0                    1.0   \n",
       "4                                 0.0                    1.0   \n",
       "\n",
       "   copy_number_low_copy  copy_number_unknown  ...  species_budding_yeast  \\\n",
       "0                   0.0                  1.0  ...                    0.0   \n",
       "1                   0.0                  0.0  ...                    0.0   \n",
       "2                   0.0                  1.0  ...                    0.0   \n",
       "3                   0.0                  0.0  ...                    0.0   \n",
       "4                   0.0                  0.0  ...                    0.0   \n",
       "\n",
       "   species_fly  species_human  species_mouse  species_mustard_weed  \\\n",
       "0          0.0            0.0            0.0                   0.0   \n",
       "1          0.0            1.0            0.0                   0.0   \n",
       "2          0.0            0.0            0.0                   0.0   \n",
       "3          0.0            1.0            0.0                   0.0   \n",
       "4          0.0            1.0            0.0                   0.0   \n",
       "\n",
       "   species_nematode  species_other  species_rat  species_synthetic  \\\n",
       "0               0.0            0.0          0.0                0.0   \n",
       "1               0.0            0.0          0.0                0.0   \n",
       "2               0.0            0.0          0.0                0.0   \n",
       "3               0.0            0.0          0.0                0.0   \n",
       "4               0.0            0.0          0.0                0.0   \n",
       "\n",
       "   species_zebrafish  \n",
       "0                1.0  \n",
       "1                0.0  \n",
       "2                0.0  \n",
       "3                0.0  \n",
       "4                0.0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "test_values = pd.read_csv(test_values_path)\n",
    "print(\"shape of test_labels: \", test_values.shape)\n",
    "test_values.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating training, val and test sets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adding columns with lab_id and sequence lengths to train_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_columns(df_values, df_labels=None):\n",
    "    lab_ids = []\n",
    "    seq_lengths = []\n",
    "    pbar = tqdm(df_values.index)\n",
    "    df_values = df_values.copy()\n",
    "    if df_labels is None:\n",
    "        lab_id = None\n",
    "    for ix in pbar:\n",
    "        if df_labels is not None:\n",
    "            lab_id = df_labels.columns[df_labels.loc[ix,:]==1][0]\n",
    "        seq_id = df_values.loc[ix,\"sequence_id\"]\n",
    "        seq_len = len(df_values.loc[ix,\"sequence\"])\n",
    "        pbar.set_description(f\"Sequence: {seq_id} Lab: {lab_id}\")\n",
    "        lab_ids.append(lab_id)\n",
    "        seq_lengths.append(seq_len)\n",
    "    cols = list(df_values.columns)[df_values.columns.get_loc(\"bacterial_resistance_ampicillin\"):]\n",
    "    df_values[\"lab_id\"] = lab_ids\n",
    "    df_values[\"seq_length\"] = seq_lengths\n",
    "    cols = [\"sequence_id\", \"lab_id\", \"sequence\", \"seq_length\"] + cols\n",
    "    df_values = df_values.loc[:,cols]\n",
    "    return df_values\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating training and test data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Sequence: U5MR3 Lab: 3EZXYI3U: 100%|██████████| 63017/63017 [01:25<00:00, 736.29it/s]\n",
      "Sequence: IAZLY Lab: None:   7%|▋         | 1302/18816 [00:00<00:07, 2500.79it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Sequence: HRMSG Lab: None:  22%|██▏       | 4071/18816 [00:01<00:05, 2911.42it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Sequence: LASAX Lab: None:  41%|████      | 7726/18816 [00:02<00:03, 3113.44it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Sequence: C2G7C Lab: None:  54%|█████▍    | 10148/18816 [00:03<00:03, 2534.32it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Sequence: UR56J Lab: None:  75%|███████▍  | 14050/18816 [00:05<00:01, 3250.82it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Sequence: AXHK0 Lab: None:  87%|████████▋ | 16376/18816 [00:05<00:00, 2663.23it/s]IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n",
      "Sequence: BD9BA Lab: None: 100%|██████████| 18816/18816 [00:06<00:00, 2740.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 36s, sys: 5.77 s, total: 1min 42s\n",
      "Wall time: 1min 32s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_train_val = add_columns(train_values,train_labels)\n",
    "df_test = add_columns(test_values,None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of df_train_val:  (63017, 43)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>lab_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>seq_length</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>CATGCATTAGTTATTAATAGTAATCAATTACGGGGTCATTAGTTCA...</td>\n",
       "      <td>7151</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5SAQC</td>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>GCTGGATGGTTTGGGACATGTGCAGCCCCGTCTCTGTATGGAGTGA...</td>\n",
       "      <td>456</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E7QRO</td>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>NNCCGGGCTGTAGCTACACAGGGCGGAGATGAGAGCCCTACGAAAG...</td>\n",
       "      <td>1450</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CT5FP</td>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>GCGGAGATGAAGAGCCCTACGAAAGCTGAGCCTGCGACTCCCGCAG...</td>\n",
       "      <td>914</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7PTD8</td>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>CGCGCATTACTTCACATGGTCCTCAAGGGTAACATGAAAGTGATCC...</td>\n",
       "      <td>1350</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id    lab_id                                           sequence  \\\n",
       "0       9ZIMC  RYUA3GVO  CATGCATTAGTTATTAATAGTAATCAATTACGGGGTCATTAGTTCA...   \n",
       "1       5SAQC  RYUA3GVO  GCTGGATGGTTTGGGACATGTGCAGCCCCGTCTCTGTATGGAGTGA...   \n",
       "2       E7QRO  RYUA3GVO  NNCCGGGCTGTAGCTACACAGGGCGGAGATGAGAGCCCTACGAAAG...   \n",
       "3       CT5FP  RYUA3GVO  GCGGAGATGAAGAGCCCTACGAAAGCTGAGCCTGCGACTCCCGCAG...   \n",
       "4       7PTD8  RYUA3GVO  CGCGCATTACTTCACATGGTCCTCAAGGGTAACATGAAAGTGATCC...   \n",
       "\n",
       "   seq_length  bacterial_resistance_ampicillin  \\\n",
       "0        7151                              0.0   \n",
       "1         456                              1.0   \n",
       "2        1450                              1.0   \n",
       "3         914                              1.0   \n",
       "4        1350                              1.0   \n",
       "\n",
       "   bacterial_resistance_chloramphenicol  bacterial_resistance_kanamycin  \\\n",
       "0                                   0.0                             1.0   \n",
       "1                                   0.0                             0.0   \n",
       "2                                   0.0                             0.0   \n",
       "3                                   0.0                             0.0   \n",
       "4                                   0.0                             0.0   \n",
       "\n",
       "   bacterial_resistance_other  bacterial_resistance_spectinomycin  \\\n",
       "0                         0.0                                 0.0   \n",
       "1                         0.0                                 0.0   \n",
       "2                         0.0                                 0.0   \n",
       "3                         0.0                                 0.0   \n",
       "4                         0.0                                 0.0   \n",
       "\n",
       "   copy_number_high_copy  ...  species_budding_yeast  species_fly  \\\n",
       "0                    1.0  ...                    0.0          0.0   \n",
       "1                    1.0  ...                    0.0          0.0   \n",
       "2                    1.0  ...                    0.0          0.0   \n",
       "3                    1.0  ...                    0.0          0.0   \n",
       "4                    1.0  ...                    0.0          0.0   \n",
       "\n",
       "   species_human  species_mouse  species_mustard_weed  species_nematode  \\\n",
       "0            0.0            1.0                   0.0               0.0   \n",
       "1            0.0            1.0                   0.0               0.0   \n",
       "2            0.0            1.0                   0.0               0.0   \n",
       "3            0.0            1.0                   0.0               0.0   \n",
       "4            0.0            1.0                   0.0               0.0   \n",
       "\n",
       "   species_other  species_rat  species_synthetic  species_zebrafish  \n",
       "0            0.0          0.0                0.0                0.0  \n",
       "1            0.0          0.0                0.0                0.0  \n",
       "2            0.0          0.0                0.0                0.0  \n",
       "3            0.0          0.0                0.0                0.0  \n",
       "4            0.0          0.0                0.0                0.0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Shape of df_train_val: \", df_train_val.shape)\n",
    "df_train_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of df_test:  (18816, 43)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>lab_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>seq_length</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>E0VFT</td>\n",
       "      <td>None</td>\n",
       "      <td>AGATCTATACATTGAATCAATATTGGCAATTAGCCATATTAGTCAT...</td>\n",
       "      <td>9379</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TTRK5</td>\n",
       "      <td>None</td>\n",
       "      <td>GCGCGCGTTGACATTGATTATTGACTAGTTATTAATAGTAATCAAT...</td>\n",
       "      <td>6673</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2Z7FZ</td>\n",
       "      <td>None</td>\n",
       "      <td>GCTTAAGCGGTCGACGGATCGGGAGATCTCCCGATCCCCTATGGTG...</td>\n",
       "      <td>9044</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VJI6E</td>\n",
       "      <td>None</td>\n",
       "      <td>ATGATGATGATGTCCCTGAACAGCAAGCAGGCGTTTAGCATGCCGC...</td>\n",
       "      <td>1230</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>721FI</td>\n",
       "      <td>None</td>\n",
       "      <td>GGTACCGAGCTCTTACGCGTGCTAGCCATACTATCAGCCACTTGTG...</td>\n",
       "      <td>6378</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id lab_id                                           sequence  \\\n",
       "0       E0VFT   None  AGATCTATACATTGAATCAATATTGGCAATTAGCCATATTAGTCAT...   \n",
       "1       TTRK5   None  GCGCGCGTTGACATTGATTATTGACTAGTTATTAATAGTAATCAAT...   \n",
       "2       2Z7FZ   None  GCTTAAGCGGTCGACGGATCGGGAGATCTCCCGATCCCCTATGGTG...   \n",
       "3       VJI6E   None  ATGATGATGATGTCCCTGAACAGCAAGCAGGCGTTTAGCATGCCGC...   \n",
       "4       721FI   None  GGTACCGAGCTCTTACGCGTGCTAGCCATACTATCAGCCACTTGTG...   \n",
       "\n",
       "   seq_length  bacterial_resistance_ampicillin  \\\n",
       "0        9379                              1.0   \n",
       "1        6673                              0.0   \n",
       "2        9044                              1.0   \n",
       "3        1230                              0.0   \n",
       "4        6378                              1.0   \n",
       "\n",
       "   bacterial_resistance_chloramphenicol  bacterial_resistance_kanamycin  \\\n",
       "0                                   0.0                             0.0   \n",
       "1                                   0.0                             0.0   \n",
       "2                                   0.0                             0.0   \n",
       "3                                   0.0                             0.0   \n",
       "4                                   0.0                             0.0   \n",
       "\n",
       "   bacterial_resistance_other  bacterial_resistance_spectinomycin  \\\n",
       "0                         0.0                                 0.0   \n",
       "1                         1.0                                 0.0   \n",
       "2                         0.0                                 0.0   \n",
       "3                         0.0                                 1.0   \n",
       "4                         0.0                                 0.0   \n",
       "\n",
       "   copy_number_high_copy  ...  species_budding_yeast  species_fly  \\\n",
       "0                    0.0  ...                    0.0          0.0   \n",
       "1                    1.0  ...                    0.0          0.0   \n",
       "2                    0.0  ...                    0.0          0.0   \n",
       "3                    1.0  ...                    0.0          0.0   \n",
       "4                    1.0  ...                    0.0          0.0   \n",
       "\n",
       "   species_human  species_mouse  species_mustard_weed  species_nematode  \\\n",
       "0            0.0            0.0                   0.0               0.0   \n",
       "1            1.0            0.0                   0.0               0.0   \n",
       "2            0.0            0.0                   0.0               0.0   \n",
       "3            1.0            0.0                   0.0               0.0   \n",
       "4            1.0            0.0                   0.0               0.0   \n",
       "\n",
       "   species_other  species_rat  species_synthetic  species_zebrafish  \n",
       "0            0.0          0.0                0.0                1.0  \n",
       "1            0.0          0.0                0.0                0.0  \n",
       "2            0.0          0.0                0.0                0.0  \n",
       "3            0.0          0.0                0.0                0.0  \n",
       "4            0.0          0.0                0.0                0.0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Shape of df_test: \", df_test.shape)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating validation set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the number of samples per lab_id is highly unbalanced. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "I7FXTVDP    8286\n",
       "RKJHZGDQ    2732\n",
       "GTVTUGVY    2672\n",
       "A18S09P2    1064\n",
       "Q2K8NHZY     973\n",
       "            ... \n",
       "58BSUZQB       3\n",
       "G2P73NZ0       3\n",
       "WB78G3XF       2\n",
       "ON9AXMKF       1\n",
       "0L3Y6ZB2       1\n",
       "Name: lab_id, Length: 1314, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_val['lab_id'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Making val data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_val(df_train_val,train_size=0.8):\n",
    "    shuffle_ix = np.random.permutation(df_train_val.index)\n",
    "    split_point = int(len(shuffle_ix)*train_size)\n",
    "    train_ix = shuffle_ix[:split_point]\n",
    "    val_ix = shuffle_ix[split_point:]\n",
    "    df_train = df_train_val.loc[train_ix,:]\n",
    "    df_val = df_train_val.loc[val_ix,:]\n",
    "    return df_train, df_val\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training set: (50413, 43) % training set: 0.7999904787596998 \n",
      "Shape of validation set: (12604, 43) % training set: 0.20000952124030025 \n",
      "CPU times: user 25.3 ms, sys: 0 ns, total: 25.3 ms\n",
      "Wall time: 24.7 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "np.random.seed(38748)\n",
    "train_size = 0.8\n",
    "df_train, df_val = make_val(df_train_val,train_size)\n",
    "print(f\"Shape of training set: {df_train.shape} % training set: {len(df_train)/len(df_train_val)} \")\n",
    "print(f\"Shape of validation set: {df_val.shape} % training set: {len(df_val)/len(df_train_val)} \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>lab_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>seq_length</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4350</th>\n",
       "      <td>SMVSS</td>\n",
       "      <td>I7FXTVDP</td>\n",
       "      <td>GACGGATCGGGAGATCTCCCGATCCCCTATGGTGCACTCTCAGTAC...</td>\n",
       "      <td>7723</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10470</th>\n",
       "      <td>6MC5M</td>\n",
       "      <td>I7FXTVDP</td>\n",
       "      <td>GCTAGAGCCGTGAACGACAGGGCGAACGCCAGCCCGCCGACGGCGA...</td>\n",
       "      <td>10550</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30997</th>\n",
       "      <td>BHOUO</td>\n",
       "      <td>P3Q11IAK</td>\n",
       "      <td>GGCTTTGTTAGCAGCCGGATCCTTATCAGTCTGCGGCAGGATTGGC...</td>\n",
       "      <td>1085</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29915</th>\n",
       "      <td>EI1J7</td>\n",
       "      <td>I7FXTVDP</td>\n",
       "      <td>GACGGATCGGGAGATCTCCCGATCCCCTATGGTGCACTCTCAGTAC...</td>\n",
       "      <td>9216</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19843</th>\n",
       "      <td>NVS4Z</td>\n",
       "      <td>AMV4U0A0</td>\n",
       "      <td>TATACGACTCACTATAGGGCGAATTGGGCCCTCTAGATGCATGCTC...</td>\n",
       "      <td>975</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 43 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      sequence_id    lab_id  \\\n",
       "4350        SMVSS  I7FXTVDP   \n",
       "10470       6MC5M  I7FXTVDP   \n",
       "30997       BHOUO  P3Q11IAK   \n",
       "29915       EI1J7  I7FXTVDP   \n",
       "19843       NVS4Z  AMV4U0A0   \n",
       "\n",
       "                                                sequence  seq_length  \\\n",
       "4350   GACGGATCGGGAGATCTCCCGATCCCCTATGGTGCACTCTCAGTAC...        7723   \n",
       "10470  GCTAGAGCCGTGAACGACAGGGCGAACGCCAGCCCGCCGACGGCGA...       10550   \n",
       "30997  GGCTTTGTTAGCAGCCGGATCCTTATCAGTCTGCGGCAGGATTGGC...        1085   \n",
       "29915  GACGGATCGGGAGATCTCCCGATCCCCTATGGTGCACTCTCAGTAC...        9216   \n",
       "19843  TATACGACTCACTATAGGGCGAATTGGGCCCTCTAGATGCATGCTC...         975   \n",
       "\n",
       "       bacterial_resistance_ampicillin  bacterial_resistance_chloramphenicol  \\\n",
       "4350                               1.0                                   0.0   \n",
       "10470                              0.0                                   0.0   \n",
       "30997                              1.0                                   0.0   \n",
       "29915                              1.0                                   0.0   \n",
       "19843                              1.0                                   0.0   \n",
       "\n",
       "       bacterial_resistance_kanamycin  bacterial_resistance_other  \\\n",
       "4350                              0.0                         0.0   \n",
       "10470                             1.0                         0.0   \n",
       "30997                             0.0                         0.0   \n",
       "29915                             0.0                         0.0   \n",
       "19843                             1.0                         0.0   \n",
       "\n",
       "       bacterial_resistance_spectinomycin  copy_number_high_copy  ...  \\\n",
       "4350                                  0.0                    0.0  ...   \n",
       "10470                                 0.0                    0.0  ...   \n",
       "30997                                 0.0                    0.0  ...   \n",
       "29915                                 0.0                    1.0  ...   \n",
       "19843                                 0.0                    1.0  ...   \n",
       "\n",
       "       species_budding_yeast  species_fly  species_human  species_mouse  \\\n",
       "4350                     0.0          0.0            0.0            0.0   \n",
       "10470                    0.0          0.0            0.0            0.0   \n",
       "30997                    0.0          0.0            0.0            0.0   \n",
       "29915                    0.0          0.0            1.0            0.0   \n",
       "19843                    0.0          0.0            0.0            0.0   \n",
       "\n",
       "       species_mustard_weed  species_nematode  species_other  species_rat  \\\n",
       "4350                    0.0               0.0            0.0          0.0   \n",
       "10470                   0.0               0.0            1.0          0.0   \n",
       "30997                   0.0               0.0            1.0          0.0   \n",
       "29915                   0.0               0.0            0.0          0.0   \n",
       "19843                   0.0               0.0            0.0          0.0   \n",
       "\n",
       "       species_synthetic  species_zebrafish  \n",
       "4350                 0.0                0.0  \n",
       "10470                0.0                0.0  \n",
       "30997                0.0                0.0  \n",
       "29915                0.0                0.0  \n",
       "19843                1.0                0.0  \n",
       "\n",
       "[5 rows x 43 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving train_df in /home/rio/data_sets/genetic_engineering_attribution/train/train.csv\n",
      "Saving train_df in /home/rio/data_sets/genetic_engineering_attribution/val/val.csv\n",
      "Saving train_df in /home/rio/data_sets/genetic_engineering_attribution/test/test.csv\n",
      "CPU times: user 5.9 s, sys: 109 ms, total: 6.01 s\n",
      "Wall time: 6.02 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_path = os.path.join(train_dir,\"train.csv\")\n",
    "val_path = os.path.join(val_dir,\"val.csv\")\n",
    "test_path = os.path.join(test_dir,\"test.csv\")\n",
    "print(f\"Saving train_df in {train_path}\")\n",
    "df_train.to_csv(train_path)\n",
    "print(f\"Saving train_df in {val_path}\")\n",
    "df_val.to_csv(val_path)\n",
    "print(f\"Saving train_df in {test_path}\")\n",
    "df_test.to_csv(test_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The code below is DRERECATED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating small samples of datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(98346)\n",
    "n_samples = 1000\n",
    "sample_train_ixs = np.random.permutation(len(train_values))[:n_samples]\n",
    "sample_test_ixs = np.random.permutation(len(test_values))[:n_samples]\n",
    "sample_train_values_path = os.path.join(sample_train_dir,\"sample_train_values.csv\")\n",
    "sample_train_labels_path = os.path.join(sample_train_dir,\"sample_train_labels.csv\")\n",
    "sample_test_values_path = os.path.join(sample_test_dir,\"sample_test_values.csv\")\n",
    "if SMALL_SAMPLE:\n",
    "    if os.path.isfile(sample_train_values_path):\n",
    "        train_values = pd.read_csv(sample_train_values_path, index_col=0)\n",
    "        train_labels = pd.read_csv(sample_train_labels_path, index_col=0)\n",
    "        test_values = pd.read_csv(sample_test_values_path, index_col=0)\n",
    "    else:\n",
    "        train_values = train_values.iloc[sample_train_ixs,:]\n",
    "        train_labels = train_labels.iloc[sample_train_ixs,:]\n",
    "        test_values = test_values.iloc[sample_test_ixs,:]\n",
    "        ### saving to csv\n",
    "        train_values.to_csv(sample_train_values_path)\n",
    "        train_labels.to_csv(sample_train_labels_path)\n",
    "        test_values.to_csv(sample_test_values_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of train_values:  (63017, 41)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>copy_number_low_copy</th>\n",
       "      <th>copy_number_unknown</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>CATGCATTAGTTATTAATAGTAATCAATTACGGGGTCATTAGTTCA...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5SAQC</td>\n",
       "      <td>GCTGGATGGTTTGGGACATGTGCAGCCCCGTCTCTGTATGGAGTGA...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E7QRO</td>\n",
       "      <td>NNCCGGGCTGTAGCTACACAGGGCGGAGATGAGAGCCCTACGAAAG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CT5FP</td>\n",
       "      <td>GCGGAGATGAAGAGCCCTACGAAAGCTGAGCCTGCGACTCCCGCAG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7PTD8</td>\n",
       "      <td>CGCGCATTACTTCACATGGTCCTCAAGGGTAACATGAAAGTGATCC...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id                                           sequence  \\\n",
       "0       9ZIMC  CATGCATTAGTTATTAATAGTAATCAATTACGGGGTCATTAGTTCA...   \n",
       "1       5SAQC  GCTGGATGGTTTGGGACATGTGCAGCCCCGTCTCTGTATGGAGTGA...   \n",
       "2       E7QRO  NNCCGGGCTGTAGCTACACAGGGCGGAGATGAGAGCCCTACGAAAG...   \n",
       "3       CT5FP  GCGGAGATGAAGAGCCCTACGAAAGCTGAGCCTGCGACTCCCGCAG...   \n",
       "4       7PTD8  CGCGCATTACTTCACATGGTCCTCAAGGGTAACATGAAAGTGATCC...   \n",
       "\n",
       "   bacterial_resistance_ampicillin  bacterial_resistance_chloramphenicol  \\\n",
       "0                              0.0                                   0.0   \n",
       "1                              1.0                                   0.0   \n",
       "2                              1.0                                   0.0   \n",
       "3                              1.0                                   0.0   \n",
       "4                              1.0                                   0.0   \n",
       "\n",
       "   bacterial_resistance_kanamycin  bacterial_resistance_other  \\\n",
       "0                             1.0                         0.0   \n",
       "1                             0.0                         0.0   \n",
       "2                             0.0                         0.0   \n",
       "3                             0.0                         0.0   \n",
       "4                             0.0                         0.0   \n",
       "\n",
       "   bacterial_resistance_spectinomycin  copy_number_high_copy  \\\n",
       "0                                 0.0                    1.0   \n",
       "1                                 0.0                    1.0   \n",
       "2                                 0.0                    1.0   \n",
       "3                                 0.0                    1.0   \n",
       "4                                 0.0                    1.0   \n",
       "\n",
       "   copy_number_low_copy  copy_number_unknown  ...  species_budding_yeast  \\\n",
       "0                   0.0                  0.0  ...                    0.0   \n",
       "1                   0.0                  0.0  ...                    0.0   \n",
       "2                   0.0                  0.0  ...                    0.0   \n",
       "3                   0.0                  0.0  ...                    0.0   \n",
       "4                   0.0                  0.0  ...                    0.0   \n",
       "\n",
       "   species_fly  species_human  species_mouse  species_mustard_weed  \\\n",
       "0          0.0            0.0            1.0                   0.0   \n",
       "1          0.0            0.0            1.0                   0.0   \n",
       "2          0.0            0.0            1.0                   0.0   \n",
       "3          0.0            0.0            1.0                   0.0   \n",
       "4          0.0            0.0            1.0                   0.0   \n",
       "\n",
       "   species_nematode  species_other  species_rat  species_synthetic  \\\n",
       "0               0.0            0.0          0.0                0.0   \n",
       "1               0.0            0.0          0.0                0.0   \n",
       "2               0.0            0.0          0.0                0.0   \n",
       "3               0.0            0.0          0.0                0.0   \n",
       "4               0.0            0.0          0.0                0.0   \n",
       "\n",
       "   species_zebrafish  \n",
       "0                0.0  \n",
       "1                0.0  \n",
       "2                0.0  \n",
       "3                0.0  \n",
       "4                0.0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"shape of train_values: \", train_values.shape)\n",
    "#print(train_values.head(10))\n",
    "display(train_values.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of train_labels:  (63017, 1315)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>00Q4V31T</th>\n",
       "      <th>012VT4JK</th>\n",
       "      <th>028IO5W2</th>\n",
       "      <th>03GRNN7N</th>\n",
       "      <th>03Y3W51H</th>\n",
       "      <th>09MQV1TY</th>\n",
       "      <th>0A4AHRCT</th>\n",
       "      <th>0A9M05NC</th>\n",
       "      <th>0B9GCUVV</th>\n",
       "      <th>...</th>\n",
       "      <th>ZQNGGY33</th>\n",
       "      <th>ZSHS4VJZ</th>\n",
       "      <th>ZT1IP3T6</th>\n",
       "      <th>ZU6860XU</th>\n",
       "      <th>ZU6TVFFU</th>\n",
       "      <th>ZU75P59K</th>\n",
       "      <th>ZUI6TDWV</th>\n",
       "      <th>ZWFD8OHC</th>\n",
       "      <th>ZX06ZDZN</th>\n",
       "      <th>ZZJVE4HO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5SAQC</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E7QRO</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CT5FP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7PTD8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1315 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id  00Q4V31T  012VT4JK  028IO5W2  03GRNN7N  03Y3W51H  09MQV1TY  \\\n",
       "0       9ZIMC       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "1       5SAQC       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "2       E7QRO       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "3       CT5FP       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "4       7PTD8       0.0       0.0       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   0A4AHRCT  0A9M05NC  0B9GCUVV  ...  ZQNGGY33  ZSHS4VJZ  ZT1IP3T6  ZU6860XU  \\\n",
       "0       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "1       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "2       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "3       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "4       0.0       0.0       0.0  ...       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   ZU6TVFFU  ZU75P59K  ZUI6TDWV  ZWFD8OHC  ZX06ZDZN  ZZJVE4HO  \n",
       "0       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "1       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "2       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "3       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "4       0.0       0.0       0.0       0.0       0.0       0.0  \n",
       "\n",
       "[5 rows x 1315 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"shape of train_labels: \", train_labels.shape)\n",
    "#print(train_values.head(10))\n",
    "display(train_labels.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of test_values:  (18816, 41)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>sequence</th>\n",
       "      <th>bacterial_resistance_ampicillin</th>\n",
       "      <th>bacterial_resistance_chloramphenicol</th>\n",
       "      <th>bacterial_resistance_kanamycin</th>\n",
       "      <th>bacterial_resistance_other</th>\n",
       "      <th>bacterial_resistance_spectinomycin</th>\n",
       "      <th>copy_number_high_copy</th>\n",
       "      <th>copy_number_low_copy</th>\n",
       "      <th>copy_number_unknown</th>\n",
       "      <th>...</th>\n",
       "      <th>species_budding_yeast</th>\n",
       "      <th>species_fly</th>\n",
       "      <th>species_human</th>\n",
       "      <th>species_mouse</th>\n",
       "      <th>species_mustard_weed</th>\n",
       "      <th>species_nematode</th>\n",
       "      <th>species_other</th>\n",
       "      <th>species_rat</th>\n",
       "      <th>species_synthetic</th>\n",
       "      <th>species_zebrafish</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>E0VFT</td>\n",
       "      <td>AGATCTATACATTGAATCAATATTGGCAATTAGCCATATTAGTCAT...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TTRK5</td>\n",
       "      <td>GCGCGCGTTGACATTGATTATTGACTAGTTATTAATAGTAATCAAT...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2Z7FZ</td>\n",
       "      <td>GCTTAAGCGGTCGACGGATCGGGAGATCTCCCGATCCCCTATGGTG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VJI6E</td>\n",
       "      <td>ATGATGATGATGTCCCTGAACAGCAAGCAGGCGTTTAGCATGCCGC...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>721FI</td>\n",
       "      <td>GGTACCGAGCTCTTACGCGTGCTAGCCATACTATCAGCCACTTGTG...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  sequence_id                                           sequence  \\\n",
       "0       E0VFT  AGATCTATACATTGAATCAATATTGGCAATTAGCCATATTAGTCAT...   \n",
       "1       TTRK5  GCGCGCGTTGACATTGATTATTGACTAGTTATTAATAGTAATCAAT...   \n",
       "2       2Z7FZ  GCTTAAGCGGTCGACGGATCGGGAGATCTCCCGATCCCCTATGGTG...   \n",
       "3       VJI6E  ATGATGATGATGTCCCTGAACAGCAAGCAGGCGTTTAGCATGCCGC...   \n",
       "4       721FI  GGTACCGAGCTCTTACGCGTGCTAGCCATACTATCAGCCACTTGTG...   \n",
       "\n",
       "   bacterial_resistance_ampicillin  bacterial_resistance_chloramphenicol  \\\n",
       "0                              1.0                                   0.0   \n",
       "1                              0.0                                   0.0   \n",
       "2                              1.0                                   0.0   \n",
       "3                              0.0                                   0.0   \n",
       "4                              1.0                                   0.0   \n",
       "\n",
       "   bacterial_resistance_kanamycin  bacterial_resistance_other  \\\n",
       "0                             0.0                         0.0   \n",
       "1                             0.0                         1.0   \n",
       "2                             0.0                         0.0   \n",
       "3                             0.0                         0.0   \n",
       "4                             0.0                         0.0   \n",
       "\n",
       "   bacterial_resistance_spectinomycin  copy_number_high_copy  \\\n",
       "0                                 0.0                    0.0   \n",
       "1                                 0.0                    1.0   \n",
       "2                                 0.0                    0.0   \n",
       "3                                 1.0                    1.0   \n",
       "4                                 0.0                    1.0   \n",
       "\n",
       "   copy_number_low_copy  copy_number_unknown  ...  species_budding_yeast  \\\n",
       "0                   0.0                  1.0  ...                    0.0   \n",
       "1                   0.0                  0.0  ...                    0.0   \n",
       "2                   0.0                  1.0  ...                    0.0   \n",
       "3                   0.0                  0.0  ...                    0.0   \n",
       "4                   0.0                  0.0  ...                    0.0   \n",
       "\n",
       "   species_fly  species_human  species_mouse  species_mustard_weed  \\\n",
       "0          0.0            0.0            0.0                   0.0   \n",
       "1          0.0            1.0            0.0                   0.0   \n",
       "2          0.0            0.0            0.0                   0.0   \n",
       "3          0.0            1.0            0.0                   0.0   \n",
       "4          0.0            1.0            0.0                   0.0   \n",
       "\n",
       "   species_nematode  species_other  species_rat  species_synthetic  \\\n",
       "0               0.0            0.0          0.0                0.0   \n",
       "1               0.0            0.0          0.0                0.0   \n",
       "2               0.0            0.0          0.0                0.0   \n",
       "3               0.0            0.0          0.0                0.0   \n",
       "4               0.0            0.0          0.0                0.0   \n",
       "\n",
       "   species_zebrafish  \n",
       "0                1.0  \n",
       "1                0.0  \n",
       "2                0.0  \n",
       "3                0.0  \n",
       "4                0.0  \n",
       "\n",
       "[5 rows x 41 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"shape of test_values: \", test_values.shape)\n",
    "#print(train_values.head(10))\n",
    "display(test_values.head(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### One-hot encoding DNA sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dna2dict(dna_seq):\n",
    "    \"\"\"\n",
    "    DNA bases are ordered in tuples alphabetically as (A,C,G,N,T)\n",
    "    \"\"\"\n",
    "    dna_dict = OrderedDict()\n",
    "    dna_seq = dna_seq.upper()\n",
    "    for ix,b in enumerate(dna_seq):\n",
    "        dna_dict[ix] = (int(b==\"A\"),int(b==\"C\"),int(b==\"G\"),int(b==\"N\"),int(b==\"T\"))\n",
    "    return dna_dict\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([(0, (0, 0, 1, 0, 0)),\n",
       "             (1, (0, 1, 0, 0, 0)),\n",
       "             (2, (0, 0, 0, 0, 1)),\n",
       "             (3, (0, 0, 1, 0, 0)),\n",
       "             (4, (0, 0, 1, 0, 0)),\n",
       "             (5, (1, 0, 0, 0, 0)),\n",
       "             (6, (0, 0, 0, 0, 1)),\n",
       "             (7, (0, 0, 1, 0, 0)),\n",
       "             (8, (0, 0, 1, 0, 0)),\n",
       "             (9, (0, 0, 0, 0, 1)),\n",
       "             (10, (0, 0, 0, 0, 1)),\n",
       "             (11, (0, 0, 0, 0, 1)),\n",
       "             (12, (0, 0, 1, 0, 0)),\n",
       "             (13, (0, 0, 1, 0, 0)),\n",
       "             (14, (0, 0, 1, 0, 0)),\n",
       "             (15, (1, 0, 0, 0, 0)),\n",
       "             (16, (0, 1, 0, 0, 0)),\n",
       "             (17, (1, 0, 0, 0, 0)),\n",
       "             (18, (0, 0, 0, 0, 1)),\n",
       "             (19, (0, 0, 1, 0, 0)),\n",
       "             (20, (0, 0, 0, 0, 1)),\n",
       "             (21, (0, 0, 1, 0, 0)),\n",
       "             (22, (0, 1, 0, 0, 0)),\n",
       "             (23, (1, 0, 0, 0, 0)),\n",
       "             (24, (0, 0, 1, 0, 0)),\n",
       "             (25, (0, 1, 0, 0, 0)),\n",
       "             (26, (0, 1, 0, 0, 0)),\n",
       "             (27, (0, 1, 0, 0, 0)),\n",
       "             (28, (0, 1, 0, 0, 0)),\n",
       "             (29, (0, 0, 1, 0, 0)),\n",
       "             (30, (0, 0, 0, 0, 1)),\n",
       "             (31, (0, 1, 0, 0, 0)),\n",
       "             (32, (0, 0, 0, 0, 1)),\n",
       "             (33, (0, 1, 0, 0, 0)),\n",
       "             (34, (0, 0, 0, 0, 1)),\n",
       "             (35, (0, 0, 1, 0, 0)),\n",
       "             (36, (0, 0, 0, 0, 1)),\n",
       "             (37, (1, 0, 0, 0, 0)),\n",
       "             (38, (0, 0, 0, 0, 1)),\n",
       "             (39, (0, 0, 1, 0, 0)),\n",
       "             (40, (0, 0, 1, 0, 0)),\n",
       "             (41, (1, 0, 0, 0, 0)),\n",
       "             (42, (0, 0, 1, 0, 0)),\n",
       "             (43, (0, 0, 0, 0, 1)),\n",
       "             (44, (0, 0, 1, 0, 0)),\n",
       "             (45, (1, 0, 0, 0, 0)),\n",
       "             (46, (0, 1, 0, 0, 0)),\n",
       "             (47, (1, 0, 0, 0, 0)),\n",
       "             (48, (0, 0, 0, 0, 1)),\n",
       "             (49, (1, 0, 0, 0, 0)),\n",
       "             (50, (0, 0, 1, 0, 0)),\n",
       "             (51, (1, 0, 0, 0, 0)),\n",
       "             (52, (0, 0, 1, 0, 0)),\n",
       "             (53, (0, 0, 0, 0, 1)),\n",
       "             (54, (0, 0, 1, 0, 0)),\n",
       "             (55, (0, 0, 0, 0, 1)),\n",
       "             (56, (0, 0, 1, 0, 0)),\n",
       "             (57, (0, 1, 0, 0, 0)),\n",
       "             (58, (0, 0, 0, 0, 1)),\n",
       "             (59, (0, 0, 1, 0, 0)),\n",
       "             (60, (0, 1, 0, 0, 0)),\n",
       "             (61, (0, 0, 0, 0, 1)),\n",
       "             (62, (0, 1, 0, 0, 0)),\n",
       "             (63, (0, 0, 0, 0, 1)),\n",
       "             (64, (0, 0, 1, 0, 0)),\n",
       "             (65, (0, 0, 1, 0, 0)),\n",
       "             (66, (0, 0, 0, 0, 1)),\n",
       "             (67, (0, 0, 0, 0, 1)),\n",
       "             (68, (0, 0, 1, 0, 0)),\n",
       "             (69, (0, 0, 1, 0, 0)),\n",
       "             (70, (0, 0, 0, 0, 1)),\n",
       "             (71, (0, 0, 1, 0, 0)),\n",
       "             (72, (1, 0, 0, 0, 0)),\n",
       "             (73, (0, 0, 1, 0, 0)),\n",
       "             (74, (0, 0, 1, 0, 0)),\n",
       "             (75, (1, 0, 0, 0, 0)),\n",
       "             (76, (0, 1, 0, 0, 0)),\n",
       "             (77, (0, 1, 0, 0, 0)),\n",
       "             (78, (1, 0, 0, 0, 0)),\n",
       "             (79, (0, 0, 1, 0, 0)),\n",
       "             (80, (0, 1, 0, 0, 0)),\n",
       "             (81, (0, 1, 0, 0, 0)),\n",
       "             (82, (0, 0, 0, 0, 1)),\n",
       "             (83, (0, 1, 0, 0, 0)),\n",
       "             (84, (0, 0, 0, 0, 1)),\n",
       "             (85, (0, 0, 0, 0, 1)),\n",
       "             (86, (0, 0, 0, 0, 1)),\n",
       "             (87, (0, 0, 1, 0, 0)),\n",
       "             (88, (0, 1, 0, 0, 0)),\n",
       "             (89, (0, 1, 0, 0, 0)),\n",
       "             (90, (0, 1, 0, 0, 0)),\n",
       "             (91, (1, 0, 0, 0, 0)),\n",
       "             (92, (0, 0, 1, 0, 0)),\n",
       "             (93, (1, 0, 0, 0, 0)),\n",
       "             (94, (0, 0, 0, 0, 1)),\n",
       "             (95, (0, 1, 0, 0, 0)),\n",
       "             (96, (0, 0, 0, 0, 1)),\n",
       "             (97, (0, 0, 0, 0, 1)),\n",
       "             (98, (0, 1, 0, 0, 0)),\n",
       "             (99, (0, 1, 0, 0, 0)),\n",
       "             (100, (0, 0, 0, 0, 1)),\n",
       "             (101, (0, 0, 1, 0, 0)),\n",
       "             (102, (1, 0, 0, 0, 0)),\n",
       "             (103, (1, 0, 0, 0, 0)),\n",
       "             (104, (0, 1, 0, 0, 0)),\n",
       "             (105, (0, 0, 0, 0, 1)),\n",
       "             (106, (0, 0, 0, 0, 1)),\n",
       "             (107, (0, 0, 1, 0, 0)),\n",
       "             (108, (1, 0, 0, 0, 0)),\n",
       "             (109, (0, 1, 0, 0, 0)),\n",
       "             (110, (0, 1, 0, 0, 0)),\n",
       "             (111, (0, 0, 0, 0, 1)),\n",
       "             (112, (0, 0, 0, 0, 1)),\n",
       "             (113, (0, 0, 0, 0, 1)),\n",
       "             (114, (0, 1, 0, 0, 0)),\n",
       "             (115, (0, 0, 0, 0, 1)),\n",
       "             (116, (0, 0, 1, 0, 0)),\n",
       "             (117, (1, 0, 0, 0, 0)),\n",
       "             (118, (1, 0, 0, 0, 0)),\n",
       "             (119, (0, 1, 0, 0, 0)),\n",
       "             (120, (0, 0, 0, 0, 1)),\n",
       "             (121, (0, 0, 0, 0, 1)),\n",
       "             (122, (0, 0, 1, 0, 0)),\n",
       "             (123, (1, 0, 0, 0, 0)),\n",
       "             (124, (0, 0, 0, 0, 1)),\n",
       "             (125, (0, 0, 1, 0, 0)),\n",
       "             (126, (0, 0, 0, 0, 1)),\n",
       "             (127, (0, 0, 1, 0, 0)),\n",
       "             (128, (1, 0, 0, 0, 0)),\n",
       "             (129, (1, 0, 0, 0, 0)),\n",
       "             (130, (0, 0, 0, 0, 1)),\n",
       "             (131, (0, 0, 1, 0, 0)),\n",
       "             (132, (1, 0, 0, 0, 0)),\n",
       "             (133, (0, 1, 0, 0, 0)),\n",
       "             (134, (0, 0, 0, 0, 1)),\n",
       "             (135, (0, 0, 0, 0, 1)),\n",
       "             (136, (0, 0, 1, 0, 0)),\n",
       "             (137, (0, 0, 1, 0, 0)),\n",
       "             (138, (1, 0, 0, 0, 0)),\n",
       "             (139, (0, 0, 0, 0, 1)),\n",
       "             (140, (1, 0, 0, 0, 0)),\n",
       "             (141, (0, 1, 0, 0, 0)),\n",
       "             (142, (1, 0, 0, 0, 0)),\n",
       "             (143, (0, 0, 1, 0, 0)),\n",
       "             (144, (1, 0, 0, 0, 0)),\n",
       "             (145, (0, 1, 0, 0, 0)),\n",
       "             (146, (1, 0, 0, 0, 0)),\n",
       "             (147, (0, 0, 1, 0, 0)),\n",
       "             (148, (0, 1, 0, 0, 0)),\n",
       "             (149, (0, 0, 0, 0, 1)),\n",
       "             (150, (0, 0, 0, 0, 1)),\n",
       "             (151, (0, 0, 0, 0, 1)),\n",
       "             (152, (0, 1, 0, 0, 0)),\n",
       "             (153, (0, 0, 0, 0, 1)),\n",
       "             (154, (0, 0, 1, 0, 0)),\n",
       "             (155, (0, 0, 1, 0, 0)),\n",
       "             (156, (0, 0, 1, 0, 0)),\n",
       "             (157, (0, 0, 0, 0, 1)),\n",
       "             (158, (0, 0, 1, 0, 0)),\n",
       "             (159, (0, 0, 1, 0, 0)),\n",
       "             (160, (1, 0, 0, 0, 0)),\n",
       "             (161, (0, 0, 0, 0, 1)),\n",
       "             (162, (0, 0, 0, 0, 1)),\n",
       "             (163, (0, 0, 1, 0, 0)),\n",
       "             (164, (1, 0, 0, 0, 0)),\n",
       "             (165, (1, 0, 0, 0, 0)),\n",
       "             (166, (0, 0, 1, 0, 0)),\n",
       "             (167, (0, 0, 0, 0, 1)),\n",
       "             (168, (0, 0, 1, 0, 0)),\n",
       "             (169, (0, 0, 1, 0, 0)),\n",
       "             (170, (0, 0, 0, 0, 1)),\n",
       "             (171, (0, 0, 1, 0, 0)),\n",
       "             (172, (0, 0, 0, 0, 1)),\n",
       "             (173, (1, 0, 0, 0, 0)),\n",
       "             (174, (0, 0, 1, 0, 0)),\n",
       "             (175, (0, 1, 0, 0, 0)),\n",
       "             (176, (0, 0, 1, 0, 0)),\n",
       "             (177, (1, 0, 0, 0, 0)),\n",
       "             (178, (0, 1, 0, 0, 0)),\n",
       "             (179, (0, 1, 0, 0, 0)),\n",
       "             (180, (1, 0, 0, 0, 0)),\n",
       "             (181, (1, 0, 0, 0, 0)),\n",
       "             (182, (0, 0, 0, 0, 1)),\n",
       "             (183, (0, 1, 0, 0, 0)),\n",
       "             (184, (0, 0, 1, 0, 0)),\n",
       "             (185, (0, 0, 1, 0, 0)),\n",
       "             (186, (1, 0, 0, 0, 0)),\n",
       "             (187, (1, 0, 0, 0, 0)),\n",
       "             (188, (1, 0, 0, 0, 0)),\n",
       "             (189, (0, 0, 0, 0, 1)),\n",
       "             (190, (0, 1, 0, 0, 0)),\n",
       "             (191, (1, 0, 0, 0, 0)),\n",
       "             (192, (0, 0, 0, 0, 1)),\n",
       "             (193, (1, 0, 0, 0, 0)),\n",
       "             (194, (0, 0, 0, 0, 1)),\n",
       "             (195, (0, 1, 0, 0, 0)),\n",
       "             (196, (0, 1, 0, 0, 0)),\n",
       "             (197, (1, 0, 0, 0, 0)),\n",
       "             (198, (1, 0, 0, 0, 0)),\n",
       "             (199, (0, 1, 0, 0, 0)),\n",
       "             (200, (0, 1, 0, 0, 0)),\n",
       "             (201, (1, 0, 0, 0, 0)),\n",
       "             (202, (0, 0, 1, 0, 0)),\n",
       "             (203, (0, 0, 0, 0, 1)),\n",
       "             (204, (1, 0, 0, 0, 0)),\n",
       "             (205, (0, 1, 0, 0, 0)),\n",
       "             (206, (1, 0, 0, 0, 0)),\n",
       "             (207, (1, 0, 0, 0, 0)),\n",
       "             (208, (0, 1, 0, 0, 0)),\n",
       "             (209, (1, 0, 0, 0, 0)),\n",
       "             (210, (1, 0, 0, 0, 0)),\n",
       "             (211, (0, 0, 0, 0, 1)),\n",
       "             (212, (0, 0, 1, 0, 0)),\n",
       "             (213, (1, 0, 0, 0, 0)),\n",
       "             (214, (0, 0, 1, 0, 0)),\n",
       "             (215, (0, 1, 0, 0, 0)),\n",
       "             (216, (0, 1, 0, 0, 0)),\n",
       "             (217, (0, 0, 0, 0, 1)),\n",
       "             (218, (0, 0, 1, 0, 0)),\n",
       "             (219, (0, 1, 0, 0, 0)),\n",
       "             (220, (0, 0, 1, 0, 0)),\n",
       "             (221, (1, 0, 0, 0, 0)),\n",
       "             (222, (1, 0, 0, 0, 0)),\n",
       "             (223, (0, 1, 0, 0, 0)),\n",
       "             (224, (1, 0, 0, 0, 0)),\n",
       "             (225, (0, 0, 0, 0, 1)),\n",
       "             (226, (1, 0, 0, 0, 0)),\n",
       "             (227, (0, 0, 0, 0, 1)),\n",
       "             (228, (0, 0, 0, 0, 1)),\n",
       "             (229, (0, 0, 0, 0, 1)),\n",
       "             (230, (0, 0, 1, 0, 0)),\n",
       "             (231, (1, 0, 0, 0, 0)),\n",
       "             (232, (1, 0, 0, 0, 0)),\n",
       "             (233, (1, 0, 0, 0, 0)),\n",
       "             (234, (1, 0, 0, 0, 0)),\n",
       "             (235, (1, 0, 0, 0, 0)),\n",
       "             (236, (1, 0, 0, 0, 0)),\n",
       "             (237, (0, 0, 0, 0, 1)),\n",
       "             (238, (1, 0, 0, 0, 0)),\n",
       "             (239, (0, 0, 1, 0, 0)),\n",
       "             (240, (1, 0, 0, 0, 0)),\n",
       "             (241, (0, 0, 0, 0, 1)),\n",
       "             (242, (0, 0, 1, 0, 0)),\n",
       "             (243, (1, 0, 0, 0, 0)),\n",
       "             (244, (1, 0, 0, 0, 0)),\n",
       "             (245, (1, 0, 0, 0, 0)),\n",
       "             (246, (1, 0, 0, 0, 0)),\n",
       "             (247, (1, 0, 0, 0, 0)),\n",
       "             (248, (1, 0, 0, 0, 0)),\n",
       "             (249, (1, 0, 0, 0, 0)),\n",
       "             (250, (0, 0, 0, 0, 1)),\n",
       "             (251, (0, 0, 1, 0, 0)),\n",
       "             (252, (1, 0, 0, 0, 0)),\n",
       "             (253, (1, 0, 0, 0, 0)),\n",
       "             (254, (0, 0, 1, 0, 0)),\n",
       "             (255, (0, 1, 0, 0, 0)),\n",
       "             (256, (1, 0, 0, 0, 0)),\n",
       "             (257, (1, 0, 0, 0, 0)),\n",
       "             (258, (1, 0, 0, 0, 0)),\n",
       "             (259, (0, 1, 0, 0, 0)),\n",
       "             (260, (0, 0, 0, 0, 1)),\n",
       "             (261, (0, 0, 0, 0, 1)),\n",
       "             (262, (0, 0, 1, 0, 0)),\n",
       "             (263, (0, 1, 0, 0, 0)),\n",
       "             (264, (0, 0, 0, 0, 1)),\n",
       "             (265, (1, 0, 0, 0, 0)),\n",
       "             (266, (0, 0, 1, 0, 0)),\n",
       "             (267, (0, 1, 0, 0, 0)),\n",
       "             (268, (0, 0, 1, 0, 0)),\n",
       "             (269, (0, 0, 1, 0, 0)),\n",
       "             (270, (0, 0, 0, 0, 1)),\n",
       "             (271, (0, 1, 0, 0, 0)),\n",
       "             (272, (0, 1, 0, 0, 0)),\n",
       "             (273, (0, 0, 0, 0, 1)),\n",
       "             (274, (0, 1, 0, 0, 0)),\n",
       "             (275, (1, 0, 0, 0, 0)),\n",
       "             (276, (0, 1, 0, 0, 0)),\n",
       "             (277, (1, 0, 0, 0, 0)),\n",
       "             (278, (0, 0, 1, 0, 0)),\n",
       "             (279, (1, 0, 0, 0, 0)),\n",
       "             (280, (1, 0, 0, 0, 0)),\n",
       "             (281, (1, 0, 0, 0, 0)),\n",
       "             (282, (0, 1, 0, 0, 0)),\n",
       "             (283, (0, 1, 0, 0, 0)),\n",
       "             (284, (0, 1, 0, 0, 0)),\n",
       "             (285, (0, 0, 0, 0, 1)),\n",
       "             (286, (0, 0, 1, 0, 0)),\n",
       "             (287, (0, 0, 1, 0, 0)),\n",
       "             (288, (1, 0, 0, 0, 0)),\n",
       "             (289, (0, 1, 0, 0, 0)),\n",
       "             (290, (1, 0, 0, 0, 0)),\n",
       "             (291, (0, 0, 1, 0, 0)),\n",
       "             (292, (0, 0, 0, 0, 1)),\n",
       "             (293, (0, 1, 0, 0, 0)),\n",
       "             (294, (0, 0, 0, 0, 1)),\n",
       "             (295, (0, 1, 0, 0, 0)),\n",
       "             (296, (0, 1, 0, 0, 0)),\n",
       "             (297, (0, 1, 0, 0, 0)),\n",
       "             (298, (0, 1, 0, 0, 0)),\n",
       "             (299, (0, 0, 1, 0, 0)),\n",
       "             (300, (0, 0, 0, 0, 1)),\n",
       "             (301, (0, 0, 1, 0, 0)),\n",
       "             (302, (0, 0, 1, 0, 0)),\n",
       "             (303, (1, 0, 0, 0, 0)),\n",
       "             (304, (0, 0, 0, 0, 1)),\n",
       "             (305, (0, 0, 1, 0, 0)),\n",
       "             (306, (1, 0, 0, 0, 0)),\n",
       "             (307, (1, 0, 0, 0, 0)),\n",
       "             (308, (0, 0, 1, 0, 0)),\n",
       "             (309, (1, 0, 0, 0, 0)),\n",
       "             (310, (0, 1, 0, 0, 0)),\n",
       "             (311, (0, 0, 1, 0, 0)),\n",
       "             (312, (0, 0, 1, 0, 0)),\n",
       "             (313, (1, 0, 0, 0, 0)),\n",
       "             (314, (0, 0, 0, 0, 1)),\n",
       "             (315, (0, 0, 0, 0, 1)),\n",
       "             (316, (0, 0, 1, 0, 0)),\n",
       "             (317, (0, 1, 0, 0, 0)),\n",
       "             (318, (0, 1, 0, 0, 0)),\n",
       "             (319, (0, 1, 0, 0, 0)),\n",
       "             (320, (0, 0, 0, 0, 1)),\n",
       "             (321, (0, 1, 0, 0, 0)),\n",
       "             (322, (1, 0, 0, 0, 0)),\n",
       "             (323, (0, 0, 0, 0, 1)),\n",
       "             (324, (0, 0, 0, 0, 1)),\n",
       "             (325, (0, 0, 0, 0, 1)),\n",
       "             (326, (0, 0, 1, 0, 0)),\n",
       "             (327, (1, 0, 0, 0, 0)),\n",
       "             (328, (0, 0, 0, 0, 1)),\n",
       "             (329, (0, 0, 1, 0, 0)),\n",
       "             (330, (0, 1, 0, 0, 0)),\n",
       "             (331, (1, 0, 0, 0, 0)),\n",
       "             (332, (0, 1, 0, 0, 0)),\n",
       "             (333, (0, 0, 0, 0, 1)),\n",
       "             (334, (0, 0, 1, 0, 0)),\n",
       "             (335, (1, 0, 0, 0, 0)),\n",
       "             (336, (0, 1, 0, 0, 0)),\n",
       "             (337, (1, 0, 0, 0, 0)),\n",
       "             (338, (0, 0, 1, 0, 0)),\n",
       "             (339, (1, 0, 0, 0, 0)),\n",
       "             (340, (0, 0, 0, 0, 1)),\n",
       "             (341, (0, 0, 1, 0, 0)),\n",
       "             (342, (0, 0, 1, 0, 0)),\n",
       "             (343, (1, 0, 0, 0, 0)),\n",
       "             (344, (0, 0, 1, 0, 0)),\n",
       "             (345, (0, 1, 0, 0, 0)),\n",
       "             (346, (0, 1, 0, 0, 0)),\n",
       "             (347, (0, 1, 0, 0, 0)),\n",
       "             (348, (0, 0, 0, 0, 1)),\n",
       "             (349, (0, 0, 1, 0, 0)),\n",
       "             (350, (1, 0, 0, 0, 0)),\n",
       "             (351, (0, 1, 0, 0, 0)),\n",
       "             (352, (0, 1, 0, 0, 0)),\n",
       "             (353, (1, 0, 0, 0, 0)),\n",
       "             (354, (0, 1, 0, 0, 0)),\n",
       "             (355, (0, 0, 0, 0, 1)),\n",
       "             (356, (0, 0, 1, 0, 0)),\n",
       "             (357, (1, 0, 0, 0, 0)),\n",
       "             (358, (0, 1, 0, 0, 0)),\n",
       "             (359, (0, 1, 0, 0, 0)),\n",
       "             (360, (1, 0, 0, 0, 0)),\n",
       "             (361, (0, 1, 0, 0, 0)),\n",
       "             (362, (0, 0, 1, 0, 0)),\n",
       "             (363, (1, 0, 0, 0, 0)),\n",
       "             (364, (1, 0, 0, 0, 0)),\n",
       "             (365, (0, 0, 1, 0, 0)),\n",
       "             (366, (0, 1, 0, 0, 0)),\n",
       "             (367, (0, 1, 0, 0, 0)),\n",
       "             (368, (1, 0, 0, 0, 0)),\n",
       "             (369, (0, 0, 1, 0, 0)),\n",
       "             (370, (0, 0, 0, 0, 1)),\n",
       "             (371, (0, 1, 0, 0, 0)),\n",
       "             (372, (0, 1, 0, 0, 0)),\n",
       "             (373, (0, 0, 0, 0, 1)),\n",
       "             (374, (0, 0, 0, 0, 1)),\n",
       "             (375, (0, 1, 0, 0, 0)),\n",
       "             (376, (0, 1, 0, 0, 0)),\n",
       "             (377, (0, 0, 0, 0, 1)),\n",
       "             (378, (0, 0, 0, 0, 1)),\n",
       "             (379, (0, 1, 0, 0, 0)),\n",
       "             (380, (1, 0, 0, 0, 0)),\n",
       "             (381, (0, 0, 0, 0, 1)),\n",
       "             (382, (0, 0, 1, 0, 0)),\n",
       "             (383, (0, 1, 0, 0, 0)),\n",
       "             (384, (0, 1, 0, 0, 0)),\n",
       "             (385, (0, 0, 0, 0, 1)),\n",
       "             (386, (0, 0, 1, 0, 0)),\n",
       "             (387, (1, 0, 0, 0, 0)),\n",
       "             (388, (1, 0, 0, 0, 0)),\n",
       "             (389, (0, 0, 1, 0, 0)),\n",
       "             (390, (0, 0, 1, 0, 0)),\n",
       "             (391, (0, 1, 0, 0, 0)),\n",
       "             (392, (1, 0, 0, 0, 0)),\n",
       "             (393, (0, 1, 0, 0, 0)),\n",
       "             (394, (0, 1, 0, 0, 0)),\n",
       "             (395, (0, 1, 0, 0, 0)),\n",
       "             (396, (0, 1, 0, 0, 0)),\n",
       "             (397, (0, 0, 0, 0, 1)),\n",
       "             (398, (0, 1, 0, 0, 0)),\n",
       "             (399, (0, 1, 0, 0, 0)),\n",
       "             (400, (0, 1, 0, 0, 0)),\n",
       "             (401, (0, 1, 0, 0, 0)),\n",
       "             (402, (0, 1, 0, 0, 0)),\n",
       "             (403, (0, 0, 0, 0, 1)),\n",
       "             (404, (0, 1, 0, 0, 0)),\n",
       "             (405, (1, 0, 0, 0, 0)),\n",
       "             (406, (1, 0, 0, 0, 0)),\n",
       "             (407, (0, 0, 1, 0, 0)),\n",
       "             (408, (1, 0, 0, 0, 0)),\n",
       "             (409, (1, 0, 0, 0, 0)),\n",
       "             (410, (0, 0, 1, 0, 0)),\n",
       "             (411, (0, 0, 1, 0, 0)),\n",
       "             (412, (1, 0, 0, 0, 0)),\n",
       "             (413, (0, 0, 1, 0, 0)),\n",
       "             (414, (1, 0, 0, 0, 0)),\n",
       "             (415, (1, 0, 0, 0, 0)),\n",
       "             (416, (1, 0, 0, 0, 0)),\n",
       "             (417, (1, 0, 0, 0, 0)),\n",
       "             (418, (1, 0, 0, 0, 0)),\n",
       "             (419, (0, 1, 0, 0, 0)),\n",
       "             (420, (0, 1, 0, 0, 0)),\n",
       "             (421, (0, 1, 0, 0, 0)),\n",
       "             (422, (0, 0, 0, 0, 1)),\n",
       "             (423, (0, 1, 0, 0, 0)),\n",
       "             (424, (0, 0, 0, 0, 1)),\n",
       "             (425, (0, 1, 0, 0, 0)),\n",
       "             (426, (0, 0, 0, 0, 1)),\n",
       "             (427, (1, 0, 0, 0, 0)),\n",
       "             (428, (0, 1, 0, 0, 0)),\n",
       "             (429, (0, 0, 0, 0, 1)),\n",
       "             (430, (0, 0, 0, 0, 1)),\n",
       "             (431, (1, 0, 0, 0, 0)),\n",
       "             (432, (1, 0, 0, 0, 0)),\n",
       "             (433, (1, 0, 0, 0, 0)),\n",
       "             (434, (1, 0, 0, 0, 0)),\n",
       "             (435, (1, 0, 0, 0, 0)),\n",
       "             (436, (0, 0, 1, 0, 0)),\n",
       "             (437, (0, 1, 0, 0, 0)),\n",
       "             (438, (0, 0, 0, 0, 1)),\n",
       "             (439, (0, 1, 0, 0, 0)),\n",
       "             (440, (0, 0, 0, 0, 1)),\n",
       "             (441, (0, 0, 0, 0, 1)),\n",
       "             (442, (1, 0, 0, 0, 0)),\n",
       "             (443, (0, 1, 0, 0, 0)),\n",
       "             (444, (0, 0, 0, 0, 1)),\n",
       "             (445, (0, 0, 1, 0, 0)),\n",
       "             (446, (0, 0, 1, 0, 0)),\n",
       "             (447, (0, 0, 1, 0, 0)),\n",
       "             (448, (0, 1, 0, 0, 0)),\n",
       "             (449, (0, 1, 0, 0, 0)),\n",
       "             (450, (0, 1, 0, 0, 0)),\n",
       "             (451, (1, 0, 0, 0, 0)),\n",
       "             (452, (1, 0, 0, 0, 0)),\n",
       "             (453, (0, 1, 0, 0, 0)),\n",
       "             (454, (0, 1, 0, 0, 0)),\n",
       "             (455, (0, 1, 0, 0, 0))])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dna_dict = dna2dict(train_values.sequence.values[1])\n",
    "dna_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "456"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dna_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 0, 0, 1),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 0, 0, 0, 1),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 0, 1, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (1, 0, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0),\n",
       " (0, 1, 0, 0, 0)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(dna_dict.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### One-hot encoded DNA sequences as dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dna2df(dna_seq,seq_id=None, lab_id = None):\n",
    "    dna_dict = dna2dict(dna_seq)\n",
    "    dna_df = pd.DataFrame(dna_dict.values(),columns=(\"A\",\"C\",\"G\",\"N\",\"T\"))\n",
    "    if seq_id:\n",
    "        columns = [\"sequence_id\"] + list(dna_df.columns)\n",
    "        dna_df[\"sequence_id\"] = [seq_id]*len(dna_df)\n",
    "        dna_df = dna_df.loc[:,columns]\n",
    "    if lab_id:\n",
    "        columns = [\"lab_id\"] + list(dna_df.columns)\n",
    "        dna_df[\"lab_id\"] = [lab_id]*len(dna_df)\n",
    "        dna_df = dna_df.loc[:,columns]\n",
    "    return dna_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lab_id</th>\n",
       "      <th>sequence_id</th>\n",
       "      <th>A</th>\n",
       "      <th>C</th>\n",
       "      <th>G</th>\n",
       "      <th>N</th>\n",
       "      <th>T</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7146</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7147</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7148</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7149</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7150</th>\n",
       "      <td>RYUA3GVO</td>\n",
       "      <td>9ZIMC</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7151 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        lab_id sequence_id  A  C  G  N  T\n",
       "0     RYUA3GVO       9ZIMC  0  1  0  0  0\n",
       "1     RYUA3GVO       9ZIMC  1  0  0  0  0\n",
       "2     RYUA3GVO       9ZIMC  0  0  0  0  1\n",
       "3     RYUA3GVO       9ZIMC  0  0  1  0  0\n",
       "4     RYUA3GVO       9ZIMC  0  1  0  0  0\n",
       "...        ...         ... .. .. .. .. ..\n",
       "7146  RYUA3GVO       9ZIMC  1  0  0  0  0\n",
       "7147  RYUA3GVO       9ZIMC  0  1  0  0  0\n",
       "7148  RYUA3GVO       9ZIMC  0  1  0  0  0\n",
       "7149  RYUA3GVO       9ZIMC  0  0  1  0  0\n",
       "7150  RYUA3GVO       9ZIMC  0  1  0  0  0\n",
       "\n",
       "[7151 rows x 7 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dna_seq = train_values.sequence.values[0]\n",
    "seq_id = train_values.sequence_id.values[0]\n",
    "lab_id = train_labels.columns[train_labels.iloc[0,:]==1][0]\n",
    "dna_df = dna2df(dna_seq=dna_seq,seq_id=seq_id,lab_id=lab_id)\n",
    "dna_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'RYUA3GVO'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lab_id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating separate csvs for each DNA sequence\n",
    "\n",
    "The .csvs are grouped by lab in different dirs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lab_id(ix,labels):\n",
    "    unique_labs = labels.columns[1:] \n",
    "    lab_id = unique_labs[labels.iloc[ix,1:]==1]\n",
    "    return lab_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10 µs, sys: 2 µs, total: 12 µs\n",
      "Wall time: 16.9 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#labs_train_dir = os.path.join(processed_train_dir,\"labs_train_dir\")\n",
    "def split_labs(df_values,df_labels=None,savedir=None,usecols=None):\n",
    "    #pbar = tqdm(train_values.iterrows(),total=len(train_values))\n",
    "    if usecols is None:\n",
    "        usecols = list(df_values.columns[df_values.columns.get_loc(\"sequence\")+1:])\n",
    "    ixs = df_values.index\n",
    "    lab_id = None\n",
    "    pbar = tqdm(ixs)\n",
    "    #print(\"len(ixs): \", len(ixs))\n",
    "    for ix in pbar:\n",
    "        row = df_values.loc[ix,:]\n",
    "        dna_seq = row.sequence\n",
    "        seq_id = row.sequence_id\n",
    "        #print(\"row: \", row)\n",
    "        if df_labels is not None:\n",
    "            lab_id = df_labels.columns[df_labels.loc[ix,:]==1][0]\n",
    "            #print(\"lab_id: \", lab_id)\n",
    "        pbar.set_description(f\"Processing sequence_id: {seq_id} lab_id: {lab_id}\")\n",
    "        dna_df = dna2df(dna_seq,seq_id,lab_id)\n",
    "        #pbar.set_description(f\"Processing sample {seq_id}\")\n",
    "        #print(\"dna_df: \", dna_df)\n",
    "        if usecols:\n",
    "            repeat = row.loc[usecols].values.reshape(1,-1)\n",
    "            repeat = np.repeat( repeat,len(dna_df), axis=0)\n",
    "            #print(\"shape: \", np.repeat(sample.loc[usecols].values,len(dna_df),axis=1).shape)\n",
    "            #print(np.repeat( sample.loc[usecols].values,len(dna_df)  ))\n",
    "            extra_cols_df = pd.DataFrame( repeat ,columns=usecols)   \n",
    "            dna_df = pd.concat([dna_df,extra_cols_df],axis=1)\n",
    "            #df_lab_concat.append(dna_df)\n",
    "            #df_lab = pd.concat(df_lab_concat,axis=0,ignore_index=True)\n",
    "        if savedir is not None:\n",
    "            if df_labels is not None:\n",
    "                labdir =os.path.join(savedir,lab_id)\n",
    "                if not os.path.isdir(labdir):\n",
    "                    os.mkdir(labdir)\n",
    "                savepath = os.path.join(labdir,seq_id+\".csv\")\n",
    "            else:\n",
    "                savepath = os.path.join(savedir,seq_id+\".csv\")\n",
    "            #print(\"savepath: \", savepath)\n",
    "            dna_df.to_csv(savepath,index=True)\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "  \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving a .csv for each lab (TRAINING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing sequence_id: U5MR3 lab_id: 3EZXYI3U: 100%|██████████| 63017/63017 [1:14:16<00:00, 14.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 12min 54s, sys: 1min 17s, total: 1h 14min 12s\n",
      "Wall time: 1h 14min 16s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "split_labs(train_values,train_labels,labs_train_dir,usecols=None)\n",
    "#split_labs(train_values,train_labels,labs_train_dir,usecols=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Saving a .csv for each seq (TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing sequence_id: BD9BA lab_id: None: 100%|██████████| 18816/18816 [22:50<00:00, 13.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 22min 32s, sys: 24 s, total: 22min 56s\n",
      "Wall time: 22min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "split_labs(test_values,None,seqs_test_dir,usecols=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:genetic_attribution]",
   "language": "python",
   "name": "conda-env-genetic_attribution-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}