assay_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assay Selection\n",
    "\n",
    "In this notebook, we prepare data for our new future directions:\n",
    "\n",
    "- Use median image CellProfiler features as a baseline model\n",
    "- Good models only need to perform well on certain assays\n",
    "- Compound diversity is important across training and testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Median Feature Extraction\n",
    "\n",
    "In this section, we can aggregate cell level features to image level by computing the median."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from json import load, dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Metadata_Plate</th>\n",
       "      <th>Metadata_Well</th>\n",
       "      <th>Metadata_Assay_Plate_Barcode</th>\n",
       "      <th>Metadata_Plate_Map_Name</th>\n",
       "      <th>Metadata_well_position</th>\n",
       "      <th>Metadata_ASSAY_WELL_ROLE</th>\n",
       "      <th>Metadata_broad_sample</th>\n",
       "      <th>Metadata_mmoles_per_liter</th>\n",
       "      <th>Metadata_solvent</th>\n",
       "      <th>Metadata_pert_id</th>\n",
       "      <th>...</th>\n",
       "      <th>Nuclei_Texture_Variance_DNA_5_0</th>\n",
       "      <th>Nuclei_Texture_Variance_ER_10_0</th>\n",
       "      <th>Nuclei_Texture_Variance_ER_3_0</th>\n",
       "      <th>Nuclei_Texture_Variance_ER_5_0</th>\n",
       "      <th>Nuclei_Texture_Variance_Mito_10_0</th>\n",
       "      <th>Nuclei_Texture_Variance_Mito_3_0</th>\n",
       "      <th>Nuclei_Texture_Variance_Mito_5_0</th>\n",
       "      <th>Nuclei_Texture_Variance_RNA_10_0</th>\n",
       "      <th>Nuclei_Texture_Variance_RNA_3_0</th>\n",
       "      <th>Nuclei_Texture_Variance_RNA_5_0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>26795</td>\n",
       "      <td>a01</td>\n",
       "      <td>26795</td>\n",
       "      <td>H-CBLD-004-4</td>\n",
       "      <td>a01</td>\n",
       "      <td>treated</td>\n",
       "      <td>BRD-K08867654-001-01-6</td>\n",
       "      <td>4.961919</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>BRD-K08867654</td>\n",
       "      <td>...</td>\n",
       "      <td>3.200403</td>\n",
       "      <td>1.953486</td>\n",
       "      <td>1.941743</td>\n",
       "      <td>1.970486</td>\n",
       "      <td>1.664505</td>\n",
       "      <td>1.584247</td>\n",
       "      <td>1.586885</td>\n",
       "      <td>1.963939</td>\n",
       "      <td>2.063034</td>\n",
       "      <td>2.120759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26795</td>\n",
       "      <td>a02</td>\n",
       "      <td>26795</td>\n",
       "      <td>H-CBLD-004-4</td>\n",
       "      <td>a02</td>\n",
       "      <td>treated</td>\n",
       "      <td>BRD-K26581628-001-01-4</td>\n",
       "      <td>4.921157</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>BRD-K26581628</td>\n",
       "      <td>...</td>\n",
       "      <td>3.456203</td>\n",
       "      <td>1.867099</td>\n",
       "      <td>1.854707</td>\n",
       "      <td>1.846309</td>\n",
       "      <td>1.427015</td>\n",
       "      <td>1.423535</td>\n",
       "      <td>1.420690</td>\n",
       "      <td>2.074904</td>\n",
       "      <td>2.070163</td>\n",
       "      <td>2.115892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>26795</td>\n",
       "      <td>a03</td>\n",
       "      <td>26795</td>\n",
       "      <td>H-CBLD-004-4</td>\n",
       "      <td>a03</td>\n",
       "      <td>treated</td>\n",
       "      <td>BRD-K55775078-001-01-1</td>\n",
       "      <td>4.806274</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>BRD-K55775078</td>\n",
       "      <td>...</td>\n",
       "      <td>3.502307</td>\n",
       "      <td>1.786906</td>\n",
       "      <td>1.875530</td>\n",
       "      <td>1.890009</td>\n",
       "      <td>1.394290</td>\n",
       "      <td>1.462195</td>\n",
       "      <td>1.436593</td>\n",
       "      <td>2.163032</td>\n",
       "      <td>2.173836</td>\n",
       "      <td>2.232259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>26795</td>\n",
       "      <td>a04</td>\n",
       "      <td>26795</td>\n",
       "      <td>H-CBLD-004-4</td>\n",
       "      <td>a04</td>\n",
       "      <td>treated</td>\n",
       "      <td>BRD-K75499899-001-01-1</td>\n",
       "      <td>4.907432</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>BRD-K75499899</td>\n",
       "      <td>...</td>\n",
       "      <td>3.923080</td>\n",
       "      <td>2.059223</td>\n",
       "      <td>2.140274</td>\n",
       "      <td>2.145159</td>\n",
       "      <td>1.670193</td>\n",
       "      <td>1.745242</td>\n",
       "      <td>1.724127</td>\n",
       "      <td>2.355939</td>\n",
       "      <td>2.302446</td>\n",
       "      <td>2.341100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>26795</td>\n",
       "      <td>a05</td>\n",
       "      <td>26795</td>\n",
       "      <td>H-CBLD-004-4</td>\n",
       "      <td>a05</td>\n",
       "      <td>treated</td>\n",
       "      <td>BRD-K14998923-001-01-6</td>\n",
       "      <td>5.029488</td>\n",
       "      <td>DMSO</td>\n",
       "      <td>BRD-K14998923</td>\n",
       "      <td>...</td>\n",
       "      <td>3.506867</td>\n",
       "      <td>1.866429</td>\n",
       "      <td>1.815833</td>\n",
       "      <td>1.827246</td>\n",
       "      <td>1.457231</td>\n",
       "      <td>1.474503</td>\n",
       "      <td>1.454954</td>\n",
       "      <td>2.159988</td>\n",
       "      <td>2.165450</td>\n",
       "      <td>2.231840</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1800 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Metadata_Plate Metadata_Well  Metadata_Assay_Plate_Barcode  \\\n",
       "0           26795           a01                         26795   \n",
       "1           26795           a02                         26795   \n",
       "2           26795           a03                         26795   \n",
       "3           26795           a04                         26795   \n",
       "4           26795           a05                         26795   \n",
       "\n",
       "  Metadata_Plate_Map_Name Metadata_well_position Metadata_ASSAY_WELL_ROLE  \\\n",
       "0            H-CBLD-004-4                    a01                  treated   \n",
       "1            H-CBLD-004-4                    a02                  treated   \n",
       "2            H-CBLD-004-4                    a03                  treated   \n",
       "3            H-CBLD-004-4                    a04                  treated   \n",
       "4            H-CBLD-004-4                    a05                  treated   \n",
       "\n",
       "    Metadata_broad_sample  Metadata_mmoles_per_liter Metadata_solvent  \\\n",
       "0  BRD-K08867654-001-01-6                   4.961919             DMSO   \n",
       "1  BRD-K26581628-001-01-4                   4.921157             DMSO   \n",
       "2  BRD-K55775078-001-01-1                   4.806274             DMSO   \n",
       "3  BRD-K75499899-001-01-1                   4.907432             DMSO   \n",
       "4  BRD-K14998923-001-01-6                   5.029488             DMSO   \n",
       "\n",
       "  Metadata_pert_id               ...                 \\\n",
       "0    BRD-K08867654               ...                  \n",
       "1    BRD-K26581628               ...                  \n",
       "2    BRD-K55775078               ...                  \n",
       "3    BRD-K75499899               ...                  \n",
       "4    BRD-K14998923               ...                  \n",
       "\n",
       "  Nuclei_Texture_Variance_DNA_5_0 Nuclei_Texture_Variance_ER_10_0  \\\n",
       "0                        3.200403                        1.953486   \n",
       "1                        3.456203                        1.867099   \n",
       "2                        3.502307                        1.786906   \n",
       "3                        3.923080                        2.059223   \n",
       "4                        3.506867                        1.866429   \n",
       "\n",
       "   Nuclei_Texture_Variance_ER_3_0 Nuclei_Texture_Variance_ER_5_0  \\\n",
       "0                        1.941743                       1.970486   \n",
       "1                        1.854707                       1.846309   \n",
       "2                        1.875530                       1.890009   \n",
       "3                        2.140274                       2.145159   \n",
       "4                        1.815833                       1.827246   \n",
       "\n",
       "  Nuclei_Texture_Variance_Mito_10_0 Nuclei_Texture_Variance_Mito_3_0  \\\n",
       "0                          1.664505                         1.584247   \n",
       "1                          1.427015                         1.423535   \n",
       "2                          1.394290                         1.462195   \n",
       "3                          1.670193                         1.745242   \n",
       "4                          1.457231                         1.474503   \n",
       "\n",
       "  Nuclei_Texture_Variance_Mito_5_0  Nuclei_Texture_Variance_RNA_10_0  \\\n",
       "0                         1.586885                          1.963939   \n",
       "1                         1.420690                          2.074904   \n",
       "2                         1.436593                          2.163032   \n",
       "3                         1.724127                          2.355939   \n",
       "4                         1.454954                          2.159988   \n",
       "\n",
       "   Nuclei_Texture_Variance_RNA_3_0  Nuclei_Texture_Variance_RNA_5_0  \n",
       "0                         2.063034                         2.120759  \n",
       "1                         2.070163                         2.115892  \n",
       "2                         2.173836                         2.232259  \n",
       "3                         2.302446                         2.341100  \n",
       "4                         2.165450                         2.231840  \n",
       "\n",
       "[5 rows x 1800 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the mean table so we know the pid, wid, sid\n",
    "df = pd.read_csv('./data/test/meta_data/extracted_features/mean_well_profiles.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.connect('./data/test/meta_data/extracted_features/26795.sqlite')\n",
    "c = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cur = c.execute(\"SELECT * FROM Image\")\n",
    "#[tuple[0] for tuple in cur.description]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Step 1:** Extract `TableNumber` while dynamically considering the sites (field of views)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "pid = 26795\n",
    "wid = 'a01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'913e6d24750b2b8e9c32ac47e4cbfdde', 1, 60.0), (u'f821fbb017abc16057c50279848bdde4', 2, 33.0), (u'94424bd0f2483095a2d49b5ca510b1c1', 3, 48.0), (u'b2cd3be4aa5712affc6d074b0eed4303', 4, 80.0), (u'9bfd4e6d3eb75696d419070645ebbe89', 5, 65.0), (u'cdf2f514e10b8cb8d64b24b128b54c78', 6, 52.0)]\n"
     ]
    }
   ],
   "source": [
    "# Extract all cell features using given \n",
    "c.execute(\n",
    "    \"\"\"\n",
    "    SELECT TableNumber, ImageNumber, Image_Count_Cells\n",
    "    FROM Image\n",
    "    WHERE Image_Metadata_Plate = {} AND Image_Metadata_Well = '{}'\n",
    "    \"\"\".format(pid, wid)\n",
    ")\n",
    "\n",
    "result = c.fetchall()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Step 2:** Extract features from the Image table (no median here)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_feature_filter_name = [\n",
    "    'TableNumber', 'ImageNumber', 'ObjectNumber',\n",
    "    'Image_Group_Index', 'Image_Group_Number', 'Image_Metadata_Plate',\n",
    "    'Image_Metadata_Site', 'Image_Metadata_Well',\n",
    "    'Image_FileName_CellOutlines', 'Image_FileName_IllumAGP',\n",
    "    'Image_FileName_IllumDNA', 'Image_FileName_IllumER',\n",
    "    'Image_FileName_IllumMito', 'Image_FileName_IllumRNA',\n",
    "    'Image_FileName_NucleiOutlines', 'Image_FileName_OrigAGP',\n",
    "    'Image_FileName_OrigDNA', 'Image_FileName_OrigER',\n",
    "    'Image_FileName_OrigMito', 'Image_FileName_OrigRNA',\n",
    "    'Nuclei_AreaShape_Zernike_2_2',\n",
    "    'Nuclei_AreaShape_Zernike_3_1',\n",
    "    'Nuclei_AreaShape_Zernike_3_3',\n",
    "    'Nuclei_AreaShape_Zernike_4_0',\n",
    "    'Nuclei_AreaShape_Zernike_4_2',\n",
    "    'Nuclei_AreaShape_Zernike_4_4',\n",
    "    'Nuclei_AreaShape_Zernike_5_1', \n",
    "    'Nuclei_AreaShape_Zernike_5_3',\n",
    "    'Nuclei_AreaShape_Zernike_5_5',\n",
    "    'Nuclei_AreaShape_Zernike_6_0',\n",
    "    'Nuclei_AreaShape_Zernike_6_2',\n",
    "    'Nuclei_AreaShape_Zernike_6_4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_AGP_1of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_AGP_2of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_AGP_3of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_AGP_4of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_DNA_1of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_DNA_2of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_DNA_3of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_DNA_4of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_ER_1of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_ER_2of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_ER_3of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_ER_4of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_Mito_1of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_Mito_2of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_Mito_3of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_Mito_4of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_RNA_1of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_RNA_2of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_RNA_3of4',\n",
    "    'Nuclei_RadialDistribution_FracAtD_RNA_4of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_AGP_1of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_AGP_2of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_AGP_3of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_AGP_4of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_DNA_1of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_DNA_2of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_DNA_3of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_DNA_4of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_ER_1of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_ER_2of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_ER_3of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_ER_4of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_Mito_1of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_Mito_2of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_Mito_3of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_Mito_4of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_RNA_1of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_RNA_2of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_RNA_3of4',\n",
    "    'Nuclei_RadialDistribution_MeanFrac_RNA_4of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_AGP_1of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_AGP_2of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_AGP_3of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_AGP_4of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_DNA_1of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_DNA_2of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_DNA_3of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_DNA_4of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_ER_1of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_ER_2of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_ER_3of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_ER_4of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_Mito_1of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_Mito_2of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_Mito_3of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_Mito_4of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_RNA_1of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_RNA_2of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_RNA_3of4',\n",
    "    'Nuclei_RadialDistribution_RadialCV_RNA_4of4',\n",
    "    'Image_MD5Digest_IllumAGP',\n",
    "    'Image_MD5Digest_IllumDNA',\n",
    "    'Image_MD5Digest_IllumER',\n",
    "    'Image_MD5Digest_IllumMito',\n",
    "    'Image_MD5Digest_IllumRNA',\n",
    "    'Image_MD5Digest_OrigAGP',\n",
    "    'Image_MD5Digest_OrigDNA',\n",
    "    'Image_MD5Digest_OrigER',\n",
    "    'Image_MD5Digest_OrigMito',\n",
    "    'Image_MD5Digest_OrigRNA',\n",
    "    'Image_PathName_CellOutlines',\n",
    "    'Image_PathName_IllumAGP',\n",
    "    'Image_PathName_IllumDNA',\n",
    "    'Image_PathName_IllumER',\n",
    "    'Image_PathName_IllumMito',\n",
    "    'Image_PathName_IllumRNA',\n",
    "    'Image_PathName_NucleiOutlines',\n",
    "    'Image_PathName_OrigAGP',\n",
    "    'Image_PathName_OrigDNA',\n",
    "    'Image_PathName_OrigER',\n",
    "    'Image_PathName_OrigMito',\n",
    "    'Image_PathName_OrigRNA',\n",
    "    'Image_URL_CellOutlines',\n",
    "    'Image_URL_IllumAGP',\n",
    "    'Image_URL_IllumDNA',\n",
    "    'Image_URL_IllumER',\n",
    "    'Image_URL_IllumMito',\n",
    "    'Image_URL_IllumRNA',\n",
    "    'Image_URL_NucleiOutlines',\n",
    "    'Image_URL_OrigAGP',\n",
    "    'Image_URL_OrigDNA',\n",
    "    'Image_URL_OrigER',\n",
    "    'Image_URL_OrigMito',\n",
    "    'Image_URL_OrigRNA'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "tid = result[0][0]\n",
    "iid = result[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "583\n"
     ]
    }
   ],
   "source": [
    "# Extract all cell features using given \n",
    "c.execute(\n",
    "    \"\"\"\n",
    "    SELECT *\n",
    "    FROM Image\n",
    "    WHERE TableNumber = '{}' AND ImageNumber = {}\n",
    "    \"\"\".format(tid, iid)\n",
    ")\n",
    "\n",
    "result = c.fetchall()\n",
    "result = np.array(result[0])\n",
    "\n",
    "#print(result)\n",
    "remove_descriptions = set(image_feature_filter_name)\n",
    "\n",
    "descriptions = [i[0] for i in c.description]\n",
    "droped_c = [i for i in range(len(descriptions)) if descriptions[i] in remove_descriptions]\n",
    "result = np.delete(result, droped_c, axis=0)\n",
    "result = result.astype(float)\n",
    "print(len(result))\n",
    "\n",
    "image_feature = result\n",
    "image_name = [i for i in descriptions if i not in remove_descriptions]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Step 3:** Use `TableNumber` and `ImageNumber` to query from the other cell level tables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60, 596)\n"
     ]
    }
   ],
   "source": [
    "# Query in Cell Table\n",
    "c.execute(\n",
    "    \"\"\"\n",
    "    SELECT *\n",
    "    FROM Cells\n",
    "    WHERE TableNumber = '{}' AND ImageNumber = {}\n",
    "    \"\"\".format(tid, iid)\n",
    ")\n",
    "\n",
    "result = np.array(c.fetchall())\n",
    "\n",
    "# Drop the `TableNumber`, `ImageNumber` and `ObjectNumber` columns\n",
    "descriptions = [i[0] for i in c.description]\n",
    "remove_descriptions = set(['TableNumber', 'ImageNumber', 'ObjectNumber'])\n",
    "droped_c = [i for i in range(len(descriptions)) if descriptions[i] in remove_descriptions]\n",
    "result = np.delete(result, droped_c, axis=1)\n",
    "\n",
    "# Change the data type of result into floats\n",
    "result = result.astype(float)\n",
    "print(result.shape)\n",
    "\n",
    "# Compute the medians of cell feature for this image\n",
    "cell_median = np.median(result, axis=0)\n",
    "cell_name = [i for i in descriptions if i not in remove_descriptions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60, 582)\n"
     ]
    }
   ],
   "source": [
    "# Query in Cytoplasm Table\n",
    "c.execute(\n",
    "    \"\"\"\n",
    "    SELECT *\n",
    "    FROM Cytoplasm\n",
    "    WHERE TableNumber = '{}' AND ImageNumber = {}\n",
    "    \"\"\".format(tid, iid)\n",
    ")\n",
    "\n",
    "result = np.array(c.fetchall())\n",
    "\n",
    "# Drop the `TableNumber`, `ImageNumber` and `ObjectNumber` columns\n",
    "descriptions = [i[0] for i in c.description]\n",
    "remove_descriptions = set(['TableNumber', 'ImageNumber', 'ObjectNumber'])\n",
    "droped_c = [i for i in range(len(descriptions)) if descriptions[i] in remove_descriptions]\n",
    "result = np.delete(result, droped_c, axis=1)\n",
    "\n",
    "# Change the data type of result into floats\n",
    "result = result.astype(float)\n",
    "print(result.shape)\n",
    "\n",
    "# Compute the medians of cell feature for this image\n",
    "cytoplasm_median = np.median(result, axis=0)\n",
    "cytoplasm_name = [i for i in descriptions if i not in remove_descriptions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60, 605)\n"
     ]
    }
   ],
   "source": [
    "# Query in Nuclei Table\n",
    "c.execute(\n",
    "    \"\"\"\n",
    "    SELECT *\n",
    "    FROM Nuclei\n",
    "    WHERE TableNumber = '{}' AND ImageNumber = {}\n",
    "    \"\"\".format(tid, iid)\n",
    ")\n",
    "\n",
    "result = np.array(c.fetchall())\n",
    "\n",
    "# Drop the `TableNumber`, `ImageNumber` and `ObjectNumber` columns\n",
    "descriptions = [i[0] for i in c.description]\n",
    "remove_descriptions = set(['TableNumber', 'ImageNumber', 'ObjectNumber'])\n",
    "droped_c = [i for i in range(len(descriptions)) if descriptions[i] in remove_descriptions]\n",
    "result = np.delete(result, droped_c, axis=1)\n",
    "\n",
    "# Change the data type of result into floats\n",
    "result = result.astype(float)\n",
    "print(result.shape)\n",
    "\n",
    "# Compute the medians of cell feature for this image\n",
    "nuclei_median = np.median(result, axis=0)\n",
    "nuclei_name = [i for i in descriptions if i not in remove_descriptions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Nuclei_AreaShape_Area',\n",
       " 'Nuclei_AreaShape_Center_X',\n",
       " 'Nuclei_AreaShape_Center_Y',\n",
       " 'Nuclei_AreaShape_Compactness',\n",
       " 'Nuclei_AreaShape_Eccentricity',\n",
       " 'Nuclei_AreaShape_EulerNumber',\n",
       " 'Nuclei_AreaShape_Extent',\n",
       " 'Nuclei_AreaShape_FormFactor',\n",
       " 'Nuclei_AreaShape_MajorAxisLength',\n",
       " 'Nuclei_AreaShape_MaxFeretDiameter',\n",
       " 'Nuclei_AreaShape_MaximumRadius',\n",
       " 'Nuclei_AreaShape_MeanRadius',\n",
       " 'Nuclei_AreaShape_MedianRadius',\n",
       " 'Nuclei_AreaShape_MinFeretDiameter',\n",
       " 'Nuclei_AreaShape_MinorAxisLength',\n",
       " 'Nuclei_AreaShape_Orientation',\n",
       " 'Nuclei_AreaShape_Perimeter',\n",
       " 'Nuclei_AreaShape_Solidity',\n",
       " 'Nuclei_AreaShape_Zernike_0_0',\n",
       " 'Nuclei_AreaShape_Zernike_1_1',\n",
       " 'Nuclei_AreaShape_Zernike_2_0',\n",
       " 'Nuclei_AreaShape_Zernike_2_2',\n",
       " 'Nuclei_AreaShape_Zernike_3_1',\n",
       " 'Nuclei_AreaShape_Zernike_3_3',\n",
       " 'Nuclei_AreaShape_Zernike_4_0',\n",
       " 'Nuclei_AreaShape_Zernike_4_2',\n",
       " 'Nuclei_AreaShape_Zernike_4_4',\n",
       " 'Nuclei_AreaShape_Zernike_5_1',\n",
       " 'Nuclei_AreaShape_Zernike_5_3',\n",
       " 'Nuclei_AreaShape_Zernike_5_5',\n",
       " 'Nuclei_AreaShape_Zernike_6_0',\n",
       " 'Nuclei_AreaShape_Zernike_6_2',\n",
       " 'Nuclei_AreaShape_Zernike_6_4',\n",
       " 'Nuclei_AreaShape_Zernike_6_6',\n",
       " 'Nuclei_AreaShape_Zernike_7_1',\n",
       " 'Nuclei_AreaShape_Zernike_7_3',\n",
       " 'Nuclei_AreaShape_Zernike_7_5',\n",
       " 'Nuclei_AreaShape_Zernike_7_7',\n",
       " 'Nuclei_AreaShape_Zernike_8_0',\n",
       " 'Nuclei_AreaShape_Zernike_8_2',\n",
       " 'Nuclei_AreaShape_Zernike_8_4',\n",
       " 'Nuclei_AreaShape_Zernike_8_6',\n",
       " 'Nuclei_AreaShape_Zernike_8_8',\n",
       " 'Nuclei_AreaShape_Zernike_9_1',\n",
       " 'Nuclei_AreaShape_Zernike_9_3',\n",
       " 'Nuclei_AreaShape_Zernike_9_5',\n",
       " 'Nuclei_AreaShape_Zernike_9_7',\n",
       " 'Nuclei_AreaShape_Zernike_9_9',\n",
       " 'Nuclei_Children_Cells_Count',\n",
       " 'Nuclei_Children_Cytoplasm_Count',\n",
       " 'Nuclei_Correlation_Correlation_DNA_AGP',\n",
       " 'Nuclei_Correlation_Correlation_DNA_ER',\n",
       " 'Nuclei_Correlation_Correlation_DNA_Mito',\n",
       " 'Nuclei_Correlation_Correlation_DNA_RNA',\n",
       " 'Nuclei_Correlation_Correlation_ER_AGP',\n",
       " 'Nuclei_Correlation_Correlation_ER_RNA',\n",
       " 'Nuclei_Correlation_Correlation_Mito_AGP',\n",
       " 'Nuclei_Correlation_Correlation_Mito_ER',\n",
       " 'Nuclei_Correlation_Correlation_Mito_RNA',\n",
       " 'Nuclei_Correlation_Correlation_RNA_AGP',\n",
       " 'Nuclei_Correlation_Costes_AGP_DNA',\n",
       " 'Nuclei_Correlation_Costes_AGP_ER',\n",
       " 'Nuclei_Correlation_Costes_AGP_Mito',\n",
       " 'Nuclei_Correlation_Costes_AGP_RNA',\n",
       " 'Nuclei_Correlation_Costes_DNA_AGP',\n",
       " 'Nuclei_Correlation_Costes_DNA_ER',\n",
       " 'Nuclei_Correlation_Costes_DNA_Mito',\n",
       " 'Nuclei_Correlation_Costes_DNA_RNA',\n",
       " 'Nuclei_Correlation_Costes_ER_AGP',\n",
       " 'Nuclei_Correlation_Costes_ER_DNA',\n",
       " 'Nuclei_Correlation_Costes_ER_Mito',\n",
       " 'Nuclei_Correlation_Costes_ER_RNA',\n",
       " 'Nuclei_Correlation_Costes_Mito_AGP',\n",
       " 'Nuclei_Correlation_Costes_Mito_DNA',\n",
       " 'Nuclei_Correlation_Costes_Mito_ER',\n",
       " 'Nuclei_Correlation_Costes_Mito_RNA',\n",
       " 'Nuclei_Correlation_Costes_RNA_AGP',\n",
       " 'Nuclei_Correlation_Costes_RNA_DNA',\n",
       " 'Nuclei_Correlation_Costes_RNA_ER',\n",
       " 'Nuclei_Correlation_Costes_RNA_Mito',\n",
       " 'Nuclei_Correlation_K_AGP_DNA',\n",
       " 'Nuclei_Correlation_K_AGP_ER',\n",
       " 'Nuclei_Correlation_K_AGP_Mito',\n",
       " 'Nuclei_Correlation_K_AGP_RNA',\n",
       " 'Nuclei_Correlation_K_DNA_AGP',\n",
       " 'Nuclei_Correlation_K_DNA_ER',\n",
       " 'Nuclei_Correlation_K_DNA_Mito',\n",
       " 'Nuclei_Correlation_K_DNA_RNA',\n",
       " 'Nuclei_Correlation_K_ER_AGP',\n",
       " 'Nuclei_Correlation_K_ER_DNA',\n",
       " 'Nuclei_Correlation_K_ER_Mito',\n",
       " 'Nuclei_Correlation_K_ER_RNA',\n",
       " 'Nuclei_Correlation_K_Mito_AGP',\n",
       " 'Nuclei_Correlation_K_Mito_DNA',\n",
       " 'Nuclei_Correlation_K_Mito_ER',\n",
       " 'Nuclei_Correlation_K_Mito_RNA',\n",
       " 'Nuclei_Correlation_K_RNA_AGP',\n",
       " 'Nuclei_Correlation_K_RNA_DNA',\n",
       " 'Nuclei_Correlation_K_RNA_ER',\n",
       " 'Nuclei_Correlation_K_RNA_Mito',\n",
       " 'Nuclei_Correlation_Manders_AGP_DNA',\n",
       " 'Nuclei_Correlation_Manders_AGP_ER',\n",
       " 'Nuclei_Correlation_Manders_AGP_Mito',\n",
       " 'Nuclei_Correlation_Manders_AGP_RNA',\n",
       " 'Nuclei_Correlation_Manders_DNA_AGP',\n",
       " 'Nuclei_Correlation_Manders_DNA_ER',\n",
       " 'Nuclei_Correlation_Manders_DNA_Mito',\n",
       " 'Nuclei_Correlation_Manders_DNA_RNA',\n",
       " 'Nuclei_Correlation_Manders_ER_AGP',\n",
       " 'Nuclei_Correlation_Manders_ER_DNA',\n",
       " 'Nuclei_Correlation_Manders_ER_Mito',\n",
       " 'Nuclei_Correlation_Manders_ER_RNA',\n",
       " 'Nuclei_Correlation_Manders_Mito_AGP',\n",
       " 'Nuclei_Correlation_Manders_Mito_DNA',\n",
       " 'Nuclei_Correlation_Manders_Mito_ER',\n",
       " 'Nuclei_Correlation_Manders_Mito_RNA',\n",
       " 'Nuclei_Correlation_Manders_RNA_AGP',\n",
       " 'Nuclei_Correlation_Manders_RNA_DNA',\n",
       " 'Nuclei_Correlation_Manders_RNA_ER',\n",
       " 'Nuclei_Correlation_Manders_RNA_Mito',\n",
       " 'Nuclei_Correlation_Overlap_DNA_AGP',\n",
       " 'Nuclei_Correlation_Overlap_DNA_ER',\n",
       " 'Nuclei_Correlation_Overlap_DNA_Mito',\n",
       " 'Nuclei_Correlation_Overlap_DNA_RNA',\n",
       " 'Nuclei_Correlation_Overlap_ER_AGP',\n",
       " 'Nuclei_Correlation_Overlap_ER_RNA',\n",
       " 'Nuclei_Correlation_Overlap_Mito_AGP',\n",
       " 'Nuclei_Correlation_Overlap_Mito_ER',\n",
       " 'Nuclei_Correlation_Overlap_Mito_RNA',\n",
       " 'Nuclei_Correlation_Overlap_RNA_AGP',\n",
       " 'Nuclei_Correlation_RWC_AGP_DNA',\n",
       " 'Nuclei_Correlation_RWC_AGP_ER',\n",
       " 'Nuclei_Correlation_RWC_AGP_Mito',\n",
       " 'Nuclei_Correlation_RWC_AGP_RNA',\n",
       " 'Nuclei_Correlation_RWC_DNA_AGP',\n",
       " 'Nuclei_Correlation_RWC_DNA_ER',\n",
       " 'Nuclei_Correlation_RWC_DNA_Mito',\n",
       " 'Nuclei_Correlation_RWC_DNA_RNA',\n",
       " 'Nuclei_Correlation_RWC_ER_AGP',\n",
       " 'Nuclei_Correlation_RWC_ER_DNA',\n",
       " 'Nuclei_Correlation_RWC_ER_Mito',\n",
       " 'Nuclei_Correlation_RWC_ER_RNA',\n",
       " 'Nuclei_Correlation_RWC_Mito_AGP',\n",
       " 'Nuclei_Correlation_RWC_Mito_DNA',\n",
       " 'Nuclei_Correlation_RWC_Mito_ER',\n",
       " 'Nuclei_Correlation_RWC_Mito_RNA',\n",
       " 'Nuclei_Correlation_RWC_RNA_AGP',\n",
       " 'Nuclei_Correlation_RWC_RNA_DNA',\n",
       " 'Nuclei_Correlation_RWC_RNA_ER',\n",
       " 'Nuclei_Correlation_RWC_RNA_Mito',\n",
       " 'Nuclei_Granularity_10_AGP',\n",
       " 'Nuclei_Granularity_10_DNA',\n",
       " 'Nuclei_Granularity_10_ER',\n",
       " 'Nuclei_Granularity_10_Mito',\n",
       " 'Nuclei_Granularity_10_RNA',\n",
       " 'Nuclei_Granularity_11_AGP',\n",
       " 'Nuclei_Granularity_11_DNA',\n",
       " 'Nuclei_Granularity_11_ER',\n",
       " 'Nuclei_Granularity_11_Mito',\n",
       " 'Nuclei_Granularity_11_RNA',\n",
       " 'Nuclei_Granularity_12_AGP',\n",
       " 'Nuclei_Granularity_12_DNA',\n",
       " 'Nuclei_Granularity_12_ER',\n",
       " 'Nuclei_Granularity_12_Mito',\n",
       " 'Nuclei_Granularity_12_RNA',\n",
       " 'Nuclei_Granularity_13_AGP',\n",
       " 'Nuclei_Granularity_13_DNA',\n",
       " 'Nuclei_Granularity_13_ER',\n",
       " 'Nuclei_Granularity_13_Mito',\n",
       " 'Nuclei_Granularity_13_RNA',\n",
       " 'Nuclei_Granularity_14_AGP',\n",
       " 'Nuclei_Granularity_14_DNA',\n",
       " 'Nuclei_Granularity_14_ER',\n",
       " 'Nuclei_Granularity_14_Mito',\n",
       " 'Nuclei_Granularity_14_RNA',\n",
       " 'Nuclei_Granularity_15_AGP',\n",
       " 'Nuclei_Granularity_15_DNA',\n",
       " 'Nuclei_Granularity_15_ER',\n",
       " 'Nuclei_Granularity_15_Mito',\n",
       " 'Nuclei_Granularity_15_RNA',\n",
       " 'Nuclei_Granularity_16_AGP',\n",
       " 'Nuclei_Granularity_16_DNA',\n",
       " 'Nuclei_Granularity_16_ER',\n",
       " 'Nuclei_Granularity_16_Mito',\n",
       " 'Nuclei_Granularity_16_RNA',\n",
       " 'Nuclei_Granularity_1_AGP',\n",
       " 'Nuclei_Granularity_1_DNA',\n",
       " 'Nuclei_Granularity_1_ER',\n",
       " 'Nuclei_Granularity_1_Mito',\n",
       " 'Nuclei_Granularity_1_RNA',\n",
       " 'Nuclei_Granularity_2_AGP',\n",
       " 'Nuclei_Granularity_2_DNA',\n",
       " 'Nuclei_Granularity_2_ER',\n",
       " 'Nuclei_Granularity_2_Mito',\n",
       " 'Nuclei_Granularity_2_RNA',\n",
       " 'Nuclei_Granularity_3_AGP',\n",
       " 'Nuclei_Granularity_3_DNA',\n",
       " 'Nuclei_Granularity_3_ER',\n",
       " 'Nuclei_Granularity_3_Mito',\n",
       " 'Nuclei_Granularity_3_RNA',\n",
       " 'Nuclei_Granularity_4_AGP',\n",
       " 'Nuclei_Granularity_4_DNA',\n",
       " 'Nuclei_Granularity_4_ER',\n",
       " 'Nuclei_Granularity_4_Mito',\n",
       " 'Nuclei_Granularity_4_RNA',\n",
       " 'Nuclei_Granularity_5_AGP',\n",
       " 'Nuclei_Granularity_5_DNA',\n",
       " 'Nuclei_Granularity_5_ER',\n",
       " 'Nuclei_Granularity_5_Mito',\n",
       " 'Nuclei_Granularity_5_RNA',\n",
       " 'Nuclei_Granularity_6_AGP',\n",
       " 'Nuclei_Granularity_6_DNA',\n",
       " 'Nuclei_Granularity_6_ER',\n",
       " 'Nuclei_Granularity_6_Mito',\n",
       " 'Nuclei_Granularity_6_RNA',\n",
       " 'Nuclei_Granularity_7_AGP',\n",
       " 'Nuclei_Granularity_7_DNA',\n",
       " 'Nuclei_Granularity_7_ER',\n",
       " 'Nuclei_Granularity_7_Mito',\n",
       " 'Nuclei_Granularity_7_RNA',\n",
       " 'Nuclei_Granularity_8_AGP',\n",
       " 'Nuclei_Granularity_8_DNA',\n",
       " 'Nuclei_Granularity_8_ER',\n",
       " 'Nuclei_Granularity_8_Mito',\n",
       " 'Nuclei_Granularity_8_RNA',\n",
       " 'Nuclei_Granularity_9_AGP',\n",
       " 'Nuclei_Granularity_9_DNA',\n",
       " 'Nuclei_Granularity_9_ER',\n",
       " 'Nuclei_Granularity_9_Mito',\n",
       " 'Nuclei_Granularity_9_RNA',\n",
       " 'Nuclei_Intensity_IntegratedIntensityEdge_AGP',\n",
       " 'Nuclei_Intensity_IntegratedIntensityEdge_DNA',\n",
       " 'Nuclei_Intensity_IntegratedIntensityEdge_ER',\n",
       " 'Nuclei_Intensity_IntegratedIntensityEdge_Mito',\n",
       " 'Nuclei_Intensity_IntegratedIntensityEdge_RNA',\n",
       " 'Nuclei_Intensity_IntegratedIntensity_AGP',\n",
       " 'Nuclei_Intensity_IntegratedIntensity_DNA',\n",
       " 'Nuclei_Intensity_IntegratedIntensity_ER',\n",
       " 'Nuclei_Intensity_IntegratedIntensity_Mito',\n",
       " 'Nuclei_Intensity_IntegratedIntensity_RNA',\n",
       " 'Nuclei_Intensity_LowerQuartileIntensity_AGP',\n",
       " 'Nuclei_Intensity_LowerQuartileIntensity_DNA',\n",
       " 'Nuclei_Intensity_LowerQuartileIntensity_ER',\n",
       " 'Nuclei_Intensity_LowerQuartileIntensity_Mito',\n",
       " 'Nuclei_Intensity_LowerQuartileIntensity_RNA',\n",
       " 'Nuclei_Intensity_MADIntensity_AGP',\n",
       " 'Nuclei_Intensity_MADIntensity_DNA',\n",
       " 'Nuclei_Intensity_MADIntensity_ER',\n",
       " 'Nuclei_Intensity_MADIntensity_Mito',\n",
       " 'Nuclei_Intensity_MADIntensity_RNA',\n",
       " 'Nuclei_Intensity_MassDisplacement_AGP',\n",
       " 'Nuclei_Intensity_MassDisplacement_DNA',\n",
       " 'Nuclei_Intensity_MassDisplacement_ER',\n",
       " 'Nuclei_Intensity_MassDisplacement_Mito',\n",
       " 'Nuclei_Intensity_MassDisplacement_RNA',\n",
       " 'Nuclei_Intensity_MaxIntensityEdge_AGP',\n",
       " 'Nuclei_Intensity_MaxIntensityEdge_DNA',\n",
       " 'Nuclei_Intensity_MaxIntensityEdge_ER',\n",
       " 'Nuclei_Intensity_MaxIntensityEdge_Mito',\n",
       " 'Nuclei_Intensity_MaxIntensityEdge_RNA',\n",
       " 'Nuclei_Intensity_MaxIntensity_AGP',\n",
       " 'Nuclei_Intensity_MaxIntensity_DNA',\n",
       " 'Nuclei_Intensity_MaxIntensity_ER',\n",
       " 'Nuclei_Intensity_MaxIntensity_Mito',\n",
       " 'Nuclei_Intensity_MaxIntensity_RNA',\n",
       " 'Nuclei_Intensity_MeanIntensityEdge_AGP',\n",
       " 'Nuclei_Intensity_MeanIntensityEdge_DNA',\n",
       " 'Nuclei_Intensity_MeanIntensityEdge_ER',\n",
       " 'Nuclei_Intensity_MeanIntensityEdge_Mito',\n",
       " 'Nuclei_Intensity_MeanIntensityEdge_RNA',\n",
       " 'Nuclei_Intensity_MeanIntensity_AGP',\n",
       " 'Nuclei_Intensity_MeanIntensity_DNA',\n",
       " 'Nuclei_Intensity_MeanIntensity_ER',\n",
       " 'Nuclei_Intensity_MeanIntensity_Mito',\n",
       " 'Nuclei_Intensity_MeanIntensity_RNA',\n",
       " 'Nuclei_Intensity_MedianIntensity_AGP',\n",
       " 'Nuclei_Intensity_MedianIntensity_DNA',\n",
       " 'Nuclei_Intensity_MedianIntensity_ER',\n",
       " 'Nuclei_Intensity_MedianIntensity_Mito',\n",
       " 'Nuclei_Intensity_MedianIntensity_RNA',\n",
       " 'Nuclei_Intensity_MinIntensityEdge_AGP',\n",
       " 'Nuclei_Intensity_MinIntensityEdge_DNA',\n",
       " 'Nuclei_Intensity_MinIntensityEdge_ER',\n",
       " 'Nuclei_Intensity_MinIntensityEdge_Mito',\n",
       " 'Nuclei_Intensity_MinIntensityEdge_RNA',\n",
       " 'Nuclei_Intensity_MinIntensity_AGP',\n",
       " 'Nuclei_Intensity_MinIntensity_DNA',\n",
       " 'Nuclei_Intensity_MinIntensity_ER',\n",
       " 'Nuclei_Intensity_MinIntensity_Mito',\n",
       " 'Nuclei_Intensity_MinIntensity_RNA',\n",
       " 'Nuclei_Intensity_StdIntensityEdge_AGP',\n",
       " 'Nuclei_Intensity_StdIntensityEdge_DNA',\n",
       " 'Nuclei_Intensity_StdIntensityEdge_ER',\n",
       " 'Nuclei_Intensity_StdIntensityEdge_Mito',\n",
       " 'Nuclei_Intensity_StdIntensityEdge_RNA',\n",
       " 'Nuclei_Intensity_StdIntensity_AGP',\n",
       " 'Nuclei_Intensity_StdIntensity_DNA',\n",
       " 'Nuclei_Intensity_StdIntensity_ER',\n",
       " 'Nuclei_Intensity_StdIntensity_Mito',\n",
       " 'Nuclei_Intensity_StdIntensity_RNA',\n",
       " 'Nuclei_Intensity_UpperQuartileIntensity_AGP',\n",
       " 'Nuclei_Intensity_UpperQuartileIntensity_DNA',\n",
       " 'Nuclei_Intensity_UpperQuartileIntensity_ER',\n",
       " 'Nuclei_Intensity_UpperQuartileIntensity_Mito',\n",
       " 'Nuclei_Intensity_UpperQuartileIntensity_RNA',\n",
       " 'Nuclei_Location_CenterMassIntensity_X_AGP',\n",
       " 'Nuclei_Location_CenterMassIntensity_X_DNA',\n",
       " 'Nuclei_Location_CenterMassIntensity_X_ER',\n",
       " 'Nuclei_Location_CenterMassIntensity_X_Mito',\n",
       " 'Nuclei_Location_CenterMassIntensity_X_RNA',\n",
       " 'Nuclei_Location_CenterMassIntensity_Y_AGP',\n",
       " 'Nuclei_Location_CenterMassIntensity_Y_DNA',\n",
       " 'Nuclei_Location_CenterMassIntensity_Y_ER',\n",
       " 'Nuclei_Location_CenterMassIntensity_Y_Mito',\n",
       " 'Nuclei_Location_CenterMassIntensity_Y_RNA',\n",
       " 'Nuclei_Location_Center_X',\n",
       " 'Nuclei_Location_Center_Y',\n",
       " 'Nuclei_Location_MaxIntensity_X_AGP',\n",
       " 'Nuclei_Location_MaxIntensity_X_DNA',\n",
       " 'Nuclei_Location_MaxIntensity_X_ER',\n",
       " 'Nuclei_Location_MaxIntensity_X_Mito',\n",
       " 'Nuclei_Location_MaxIntensity_X_RNA',\n",
       " 'Nuclei_Location_MaxIntensity_Y_AGP',\n",
       " 'Nuclei_Location_MaxIntensity_Y_DNA',\n",
       " 'Nuclei_Location_MaxIntensity_Y_ER',\n",
       " 'Nuclei_Location_MaxIntensity_Y_Mito',\n",
       " 'Nuclei_Location_MaxIntensity_Y_RNA',\n",
       " 'Nuclei_Neighbors_AngleBetweenNeighbors_1',\n",
       " 'Nuclei_Neighbors_FirstClosestDistance_1',\n",
       " 'Nuclei_Neighbors_FirstClosestObjectNumber_1',\n",
       " 'Nuclei_Neighbors_NumberOfNeighbors_1',\n",
       " 'Nuclei_Neighbors_PercentTouching_1',\n",
       " 'Nuclei_Neighbors_SecondClosestDistance_1',\n",
       " 'Nuclei_Neighbors_SecondClosestObjectNumber_1',\n",
       " 'Nuclei_Number_Object_Number',\n",
       " 'Nuclei_RadialDistribution_FracAtD_AGP_1of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_AGP_2of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_AGP_3of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_AGP_4of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_DNA_1of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_DNA_2of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_DNA_3of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_DNA_4of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_ER_1of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_ER_2of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_ER_3of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_ER_4of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_Mito_1of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_Mito_2of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_Mito_3of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_Mito_4of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_RNA_1of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_RNA_2of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_RNA_3of4',\n",
       " 'Nuclei_RadialDistribution_FracAtD_RNA_4of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_AGP_1of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_AGP_2of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_AGP_3of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_AGP_4of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_DNA_1of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_DNA_2of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_DNA_3of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_DNA_4of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_ER_1of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_ER_2of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_ER_3of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_ER_4of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_Mito_1of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_Mito_2of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_Mito_3of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_Mito_4of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_RNA_1of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_RNA_2of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_RNA_3of4',\n",
       " 'Nuclei_RadialDistribution_MeanFrac_RNA_4of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_AGP_1of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_AGP_2of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_AGP_3of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_AGP_4of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_DNA_1of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_DNA_2of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_DNA_3of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_DNA_4of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_ER_1of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_ER_2of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_ER_3of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_ER_4of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_Mito_1of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_Mito_2of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_Mito_3of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_Mito_4of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_RNA_1of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_RNA_2of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_RNA_3of4',\n",
       " 'Nuclei_RadialDistribution_RadialCV_RNA_4of4',\n",
       " 'Nuclei_Texture_AngularSecondMoment_AGP_10_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_AGP_3_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_AGP_5_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_DNA_10_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_DNA_3_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_DNA_5_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_ER_10_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_ER_3_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_ER_5_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_Mito_10_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_Mito_3_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_Mito_5_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_RNA_10_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_RNA_3_0',\n",
       " 'Nuclei_Texture_AngularSecondMoment_RNA_5_0',\n",
       " 'Nuclei_Texture_Contrast_AGP_10_0',\n",
       " 'Nuclei_Texture_Contrast_AGP_3_0',\n",
       " 'Nuclei_Texture_Contrast_AGP_5_0',\n",
       " 'Nuclei_Texture_Contrast_DNA_10_0',\n",
       " 'Nuclei_Texture_Contrast_DNA_3_0',\n",
       " 'Nuclei_Texture_Contrast_DNA_5_0',\n",
       " 'Nuclei_Texture_Contrast_ER_10_0',\n",
       " 'Nuclei_Texture_Contrast_ER_3_0',\n",
       " 'Nuclei_Texture_Contrast_ER_5_0',\n",
       " 'Nuclei_Texture_Contrast_Mito_10_0',\n",
       " 'Nuclei_Texture_Contrast_Mito_3_0',\n",
       " 'Nuclei_Texture_Contrast_Mito_5_0',\n",
       " 'Nuclei_Texture_Contrast_RNA_10_0',\n",
       " 'Nuclei_Texture_Contrast_RNA_3_0',\n",
       " 'Nuclei_Texture_Contrast_RNA_5_0',\n",
       " 'Nuclei_Texture_Correlation_AGP_10_0',\n",
       " 'Nuclei_Texture_Correlation_AGP_3_0',\n",
       " 'Nuclei_Texture_Correlation_AGP_5_0',\n",
       " 'Nuclei_Texture_Correlation_DNA_10_0',\n",
       " 'Nuclei_Texture_Correlation_DNA_3_0',\n",
       " 'Nuclei_Texture_Correlation_DNA_5_0',\n",
       " 'Nuclei_Texture_Correlation_ER_10_0',\n",
       " 'Nuclei_Texture_Correlation_ER_3_0',\n",
       " 'Nuclei_Texture_Correlation_ER_5_0',\n",
       " 'Nuclei_Texture_Correlation_Mito_10_0',\n",
       " 'Nuclei_Texture_Correlation_Mito_3_0',\n",
       " 'Nuclei_Texture_Correlation_Mito_5_0',\n",
       " 'Nuclei_Texture_Correlation_RNA_10_0',\n",
       " 'Nuclei_Texture_Correlation_RNA_3_0',\n",
       " 'Nuclei_Texture_Correlation_RNA_5_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_AGP_10_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_AGP_3_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_AGP_5_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_DNA_10_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_DNA_3_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_DNA_5_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_ER_10_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_ER_3_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_ER_5_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_Mito_10_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_Mito_3_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_Mito_5_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_RNA_10_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_RNA_3_0',\n",
       " 'Nuclei_Texture_DifferenceEntropy_RNA_5_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_AGP_10_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_AGP_3_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_AGP_5_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_DNA_10_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_DNA_3_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_DNA_5_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_ER_10_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_ER_3_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_ER_5_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_Mito_10_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_Mito_3_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_Mito_5_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_RNA_10_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_RNA_3_0',\n",
       " 'Nuclei_Texture_DifferenceVariance_RNA_5_0',\n",
       " 'Nuclei_Texture_Entropy_AGP_10_0',\n",
       " 'Nuclei_Texture_Entropy_AGP_3_0',\n",
       " 'Nuclei_Texture_Entropy_AGP_5_0',\n",
       " 'Nuclei_Texture_Entropy_DNA_10_0',\n",
       " 'Nuclei_Texture_Entropy_DNA_3_0',\n",
       " 'Nuclei_Texture_Entropy_DNA_5_0',\n",
       " 'Nuclei_Texture_Entropy_ER_10_0',\n",
       " 'Nuclei_Texture_Entropy_ER_3_0',\n",
       " 'Nuclei_Texture_Entropy_ER_5_0',\n",
       " 'Nuclei_Texture_Entropy_Mito_10_0',\n",
       " 'Nuclei_Texture_Entropy_Mito_3_0',\n",
       " 'Nuclei_Texture_Entropy_Mito_5_0',\n",
       " 'Nuclei_Texture_Entropy_RNA_10_0',\n",
       " 'Nuclei_Texture_Entropy_RNA_3_0',\n",
       " 'Nuclei_Texture_Entropy_RNA_5_0',\n",
       " 'Nuclei_Texture_Gabor_AGP_10',\n",
       " 'Nuclei_Texture_Gabor_AGP_3',\n",
       " 'Nuclei_Texture_Gabor_AGP_5',\n",
       " 'Nuclei_Texture_Gabor_DNA_10',\n",
       " 'Nuclei_Texture_Gabor_DNA_3',\n",
       " 'Nuclei_Texture_Gabor_DNA_5',\n",
       " 'Nuclei_Texture_Gabor_ER_10',\n",
       " 'Nuclei_Texture_Gabor_ER_3',\n",
       " 'Nuclei_Texture_Gabor_ER_5',\n",
       " 'Nuclei_Texture_Gabor_Mito_10',\n",
       " 'Nuclei_Texture_Gabor_Mito_3',\n",
       " 'Nuclei_Texture_Gabor_Mito_5',\n",
       " 'Nuclei_Texture_Gabor_RNA_10',\n",
       " 'Nuclei_Texture_Gabor_RNA_3',\n",
       " 'Nuclei_Texture_Gabor_RNA_5',\n",
       " 'Nuclei_Texture_InfoMeas1_AGP_10_0',\n",
       " 'Nuclei_Texture_InfoMeas1_AGP_3_0',\n",
       " 'Nuclei_Texture_InfoMeas1_AGP_5_0',\n",
       " 'Nuclei_Texture_InfoMeas1_DNA_10_0',\n",
       " 'Nuclei_Texture_InfoMeas1_DNA_3_0',\n",
       " 'Nuclei_Texture_InfoMeas1_DNA_5_0',\n",
       " 'Nuclei_Texture_InfoMeas1_ER_10_0',\n",
       " 'Nuclei_Texture_InfoMeas1_ER_3_0',\n",
       " 'Nuclei_Texture_InfoMeas1_ER_5_0',\n",
       " 'Nuclei_Texture_InfoMeas1_Mito_10_0',\n",
       " 'Nuclei_Texture_InfoMeas1_Mito_3_0',\n",
       " 'Nuclei_Texture_InfoMeas1_Mito_5_0',\n",
       " 'Nuclei_Texture_InfoMeas1_RNA_10_0',\n",
       " 'Nuclei_Texture_InfoMeas1_RNA_3_0',\n",
       " 'Nuclei_Texture_InfoMeas1_RNA_5_0',\n",
       " 'Nuclei_Texture_InfoMeas2_AGP_10_0',\n",
       " 'Nuclei_Texture_InfoMeas2_AGP_3_0',\n",
       " 'Nuclei_Texture_InfoMeas2_AGP_5_0',\n",
       " 'Nuclei_Texture_InfoMeas2_DNA_10_0',\n",
       " 'Nuclei_Texture_InfoMeas2_DNA_3_0',\n",
       " 'Nuclei_Texture_InfoMeas2_DNA_5_0',\n",
       " 'Nuclei_Texture_InfoMeas2_ER_10_0',\n",
       " 'Nuclei_Texture_InfoMeas2_ER_3_0',\n",
       " 'Nuclei_Texture_InfoMeas2_ER_5_0',\n",
       " 'Nuclei_Texture_InfoMeas2_Mito_10_0',\n",
       " 'Nuclei_Texture_InfoMeas2_Mito_3_0',\n",
       " 'Nuclei_Texture_InfoMeas2_Mito_5_0',\n",
       " 'Nuclei_Texture_InfoMeas2_RNA_10_0',\n",
       " 'Nuclei_Texture_InfoMeas2_RNA_3_0',\n",
       " 'Nuclei_Texture_InfoMeas2_RNA_5_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_AGP_10_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_AGP_3_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_AGP_5_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_DNA_10_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_DNA_3_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_DNA_5_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_ER_10_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_ER_3_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_ER_5_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_Mito_10_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_Mito_3_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_Mito_5_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_RNA_10_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_RNA_3_0',\n",
       " 'Nuclei_Texture_InverseDifferenceMoment_RNA_5_0',\n",
       " 'Nuclei_Texture_SumAverage_AGP_10_0',\n",
       " 'Nuclei_Texture_SumAverage_AGP_3_0',\n",
       " 'Nuclei_Texture_SumAverage_AGP_5_0',\n",
       " 'Nuclei_Texture_SumAverage_DNA_10_0',\n",
       " 'Nuclei_Texture_SumAverage_DNA_3_0',\n",
       " 'Nuclei_Texture_SumAverage_DNA_5_0',\n",
       " 'Nuclei_Texture_SumAverage_ER_10_0',\n",
       " 'Nuclei_Texture_SumAverage_ER_3_0',\n",
       " 'Nuclei_Texture_SumAverage_ER_5_0',\n",
       " 'Nuclei_Texture_SumAverage_Mito_10_0',\n",
       " 'Nuclei_Texture_SumAverage_Mito_3_0',\n",
       " 'Nuclei_Texture_SumAverage_Mito_5_0',\n",
       " 'Nuclei_Texture_SumAverage_RNA_10_0',\n",
       " 'Nuclei_Texture_SumAverage_RNA_3_0',\n",
       " 'Nuclei_Texture_SumAverage_RNA_5_0',\n",
       " 'Nuclei_Texture_SumEntropy_AGP_10_0',\n",
       " 'Nuclei_Texture_SumEntropy_AGP_3_0',\n",
       " 'Nuclei_Texture_SumEntropy_AGP_5_0',\n",
       " 'Nuclei_Texture_SumEntropy_DNA_10_0',\n",
       " 'Nuclei_Texture_SumEntropy_DNA_3_0',\n",
       " 'Nuclei_Texture_SumEntropy_DNA_5_0',\n",
       " 'Nuclei_Texture_SumEntropy_ER_10_0',\n",
       " 'Nuclei_Texture_SumEntropy_ER_3_0',\n",
       " 'Nuclei_Texture_SumEntropy_ER_5_0',\n",
       " 'Nuclei_Texture_SumEntropy_Mito_10_0',\n",
       " 'Nuclei_Texture_SumEntropy_Mito_3_0',\n",
       " 'Nuclei_Texture_SumEntropy_Mito_5_0',\n",
       " 'Nuclei_Texture_SumEntropy_RNA_10_0',\n",
       " 'Nuclei_Texture_SumEntropy_RNA_3_0',\n",
       " 'Nuclei_Texture_SumEntropy_RNA_5_0',\n",
       " 'Nuclei_Texture_SumVariance_AGP_10_0',\n",
       " 'Nuclei_Texture_SumVariance_AGP_3_0',\n",
       " 'Nuclei_Texture_SumVariance_AGP_5_0',\n",
       " 'Nuclei_Texture_SumVariance_DNA_10_0',\n",
       " 'Nuclei_Texture_SumVariance_DNA_3_0',\n",
       " 'Nuclei_Texture_SumVariance_DNA_5_0',\n",
       " 'Nuclei_Texture_SumVariance_ER_10_0',\n",
       " 'Nuclei_Texture_SumVariance_ER_3_0',\n",
       " 'Nuclei_Texture_SumVariance_ER_5_0',\n",
       " 'Nuclei_Texture_SumVariance_Mito_10_0',\n",
       " 'Nuclei_Texture_SumVariance_Mito_3_0',\n",
       " 'Nuclei_Texture_SumVariance_Mito_5_0',\n",
       " 'Nuclei_Texture_SumVariance_RNA_10_0',\n",
       " 'Nuclei_Texture_SumVariance_RNA_3_0',\n",
       " 'Nuclei_Texture_SumVariance_RNA_5_0',\n",
       " 'Nuclei_Texture_Variance_AGP_10_0',\n",
       " 'Nuclei_Texture_Variance_AGP_3_0',\n",
       " 'Nuclei_Texture_Variance_AGP_5_0',\n",
       " 'Nuclei_Texture_Variance_DNA_10_0',\n",
       " 'Nuclei_Texture_Variance_DNA_3_0',\n",
       " 'Nuclei_Texture_Variance_DNA_5_0',\n",
       " 'Nuclei_Texture_Variance_ER_10_0',\n",
       " 'Nuclei_Texture_Variance_ER_3_0',\n",
       " 'Nuclei_Texture_Variance_ER_5_0',\n",
       " 'Nuclei_Texture_Variance_Mito_10_0',\n",
       " 'Nuclei_Texture_Variance_Mito_3_0',\n",
       " 'Nuclei_Texture_Variance_Mito_5_0',\n",
       " 'Nuclei_Texture_Variance_RNA_10_0',\n",
       " 'Nuclei_Texture_Variance_RNA_3_0',\n",
       " 'Nuclei_Texture_Variance_RNA_5_0']"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nuclei_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Store the feature names\n",
    "# For each extraction, the feature names must match this \"standard\"\n",
    "\n",
    "extract_info_dict = {\n",
    "    'image_feature_filter_name': image_feature_filter_name,\n",
    "    'image_name': image_name,\n",
    "    'cell_name': cell_name,\n",
    "    'cytoplasm_name': cytoplasm_name,\n",
    "    'nuclei_name': nuclei_name,\n",
    "    'cell_feature_filter_name': ['TableNumber', 'ImageNumber', 'ObjectNumber'],\n",
    "    'cytoplasm_feature_filter_name': ['TableNumber', 'ImageNumber', 'ObjectNumber'],\n",
    "    'nuclei_feature_filter_name': ['TableNumber', 'ImageNumber', 'ObjectNumber']\n",
    "}\n",
    "\n",
    "dump(extract_info_dict, open('./resource/assay_selection/extract_info.json', 'w'), indent=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. Script to extract features from one plate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_info_dict = load(open('./resource/assay_selection/extract_info.json', 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the mean table so we know the pid, wid, sid\n",
    "df = pd.read_csv('./data/test/meta_data/extracted_features/mean_well_profiles.csv')\n",
    "\n",
    "# Connect the sql db\n",
    "conn = sqlite3.connect('./data/test/meta_data/extracted_features/26795.sqlite')\n",
    "c = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_cell_level_feature(table, tid, iid, filter_names, validation_names):\n",
    "    \n",
    "    # Extract cell level features\n",
    "    c.execute(\n",
    "        \"\"\"\n",
    "        SELECT *\n",
    "        FROM {}\n",
    "        WHERE TableNumber = '{}' AND ImageNumber = {}\n",
    "        \"\"\".format(table, tid, iid)\n",
    "    )\n",
    "\n",
    "    result = np.array(c.fetchall())\n",
    "\n",
    "    # Filter out some features\n",
    "    descriptions = [i[0] for i in c.description]\n",
    "    droped_c = [i for i in range(len(descriptions)) if descriptions[i] in\n",
    "                filter_names]\n",
    "    result = np.delete(result, droped_c, axis=1)\n",
    "\n",
    "    # Change the data type of result into floats\n",
    "    result = result.astype(float)\n",
    "\n",
    "    # Compute the medians of cell feature for this image\n",
    "    median = np.median(result, axis=0)\n",
    "    name = [i for i in descriptions if i not in filter_names]\n",
    "    assert(name == validation_names)\n",
    "    \n",
    "    return median\n",
    "\n",
    "\n",
    "def extract_feature(c, pid, wid):\n",
    "    \n",
    "    # Compile the filtering feature names\n",
    "    image_feature_filter_names = set(extract_info_dict['image_feature_filter_name'])\n",
    "    cell_feature_filter_names = set(extract_info_dict['cell_feature_filter_name'])\n",
    "    cytoplasm_feature_filter_names = set(extract_info_dict['cytoplasm_feature_filter_name'])\n",
    "    nuclei_feature_filter_names = set(extract_info_dict['nuclei_feature_filter_name'])\n",
    "    \n",
    "    # Extract all images (different sites / fields of view)\n",
    "    c.execute(\n",
    "        \"\"\"\n",
    "        SELECT TableNumber, ImageNumber\n",
    "        FROM Image\n",
    "        WHERE Image_Metadata_Plate = {} AND Image_Metadata_Well = '{}'\n",
    "        \"\"\".format(pid, wid)\n",
    "    )\n",
    "\n",
    "    tid_iid_pairs = c.fetchall()\n",
    "    \n",
    "    # Track the feature names (row index)\n",
    "    row_index = []\n",
    "    features = []\n",
    "    \n",
    "    # Iterate through all sites and extract features from each site\n",
    "    for p in tid_iid_pairs:\n",
    "        tid, iid = p[0], p[1]\n",
    "        \n",
    "        # Extract image features\n",
    "        c.execute(\n",
    "            \"\"\"\n",
    "            SELECT *\n",
    "            FROM Image\n",
    "            WHERE TableNumber = '{}' AND ImageNumber = {}\n",
    "            \"\"\".format(tid, iid)\n",
    "        )\n",
    "\n",
    "        result = c.fetchall()\n",
    "        result = np.array(result[0])\n",
    "\n",
    "        # Filter out some features\n",
    "        descriptions = [i[0] for i in c.description]\n",
    "        droped_c = [i for i in range(len(descriptions)) if descriptions[i] in\n",
    "                    image_feature_filter_names]\n",
    "        result = np.delete(result, droped_c, axis=0)\n",
    "        \n",
    "        # Change the data type of result into floats\n",
    "        result = result.astype(float)\n",
    "\n",
    "        image_feature = result\n",
    "        image_name = [i for i in descriptions if i not in image_feature_filter_names]\n",
    "        assert(image_name == extract_info_dict['image_name'])\n",
    "        \n",
    "        # Extract cell, cytoplasm, and nuclei features\n",
    "        cell_feature = extract_cell_level_feature(\n",
    "            'Cells',\n",
    "            tid,\n",
    "            iid,\n",
    "            extract_info_dict['cell_feature_filter_name'],\n",
    "            extract_info_dict['cell_name']\n",
    "        )\n",
    "        \n",
    "        cytoplasm_feature = extract_cell_level_feature(\n",
    "            'Cytoplasm',\n",
    "            tid,\n",
    "            iid,\n",
    "            extract_info_dict['cytoplasm_feature_filter_name'],\n",
    "            extract_info_dict['cytoplasm_name']\n",
    "        )\n",
    "        \n",
    "        nuclei_feature = extract_cell_level_feature(\n",
    "            'Nuclei',\n",
    "            tid,\n",
    "            iid,\n",
    "            extract_info_dict['nuclei_feature_filter_name'],\n",
    "            extract_info_dict['nuclei_name']\n",
    "        )\n",
    "\n",
    "        # Combine image feature, cell level medians together\n",
    "        cur_feature = np.hstack((image_feature,\n",
    "                                 cell_feature,\n",
    "                                 cytoplasm_feature,\n",
    "                                 nuclei_feature))\n",
    "        \n",
    "        # Add the current feature into the well feature collections\n",
    "        features.append(cur_feature)\n",
    "        row_index.append('{}_{}_{}'.format(pid, wid, iid))\n",
    "        \n",
    "    features = np.vstack(features)\n",
    "    return features, row_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "plate_features = []\n",
    "plate_row_index = []\n",
    "\n",
    "for i, r in df.iterrows():\n",
    "    if i == 2:\n",
    "        break\n",
    "        \n",
    "    wid = r['Metadata_Plate']\n",
    "    pid = r['Metadata_Well']\n",
    "    \n",
    "    features, row_index = extract_feature(c, wid, pid)\n",
    "    plate_features.append(features)\n",
    "    plate_row_index.extend(row_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Select Assays\n",
    "\n",
    "**This section require rdkit environment**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from json import load, dump\n",
    "from rdkit import Chem\n",
    "from rdkit import DataStructs\n",
    "from rdkit.Chem.Fingerprints import FingerprintMols\n",
    "from sklearn.utils import shuffle\n",
    "import deepchem as dc\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data = np.load(\"./resource/output_matrix_convert_collision.npz\")\n",
    "output_matrix = output_data['output_matrix']\n",
    "output_inchi = output_data['compound_inchi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27241, 212)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27241,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_inchi.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_similarities(a):\n",
    "    \"\"\"\n",
    "    Compute the pairwise similarities of assay a.\n",
    "    \"\"\"\n",
    "    \n",
    "    similarities = []\n",
    "\n",
    "    # Get used compounds\n",
    "    used_compound_inchis = output_data['compound_inchi'][output_matrix[:,a] != -1]\n",
    "\n",
    "    # Convert inchi to fps\n",
    "    used_compound_fps = []\n",
    "    for i in used_compound_inchis:\n",
    "        mol = Chem.inchi.MolFromInchi(str(i))\n",
    "        used_compound_fps.append(FingerprintMols.FingerprintMol(mol))\n",
    "\n",
    "    # Compute pairwise similarities\n",
    "    for i in range(len(used_compound_fps)):\n",
    "        for j in range(i + 1, len(used_compound_fps)):\n",
    "            similarities.append(DataStructs.TanimotoSimilarity(used_compound_fps[i],\n",
    "                                                               used_compound_fps[j]))\n",
    "            \n",
    "    return similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the compound size of all assays\n",
    "assay_sizes = []\n",
    "for a in range(output_matrix.shape[1]):\n",
    "    cur_assay = output_matrix[:, a]\n",
    "    pos_size = np.sum(cur_assay == 1)\n",
    "    neg_size = np.sum(cur_assay == 0)\n",
    "    assay_sizes.append((a, pos_size + neg_size))\n",
    "\n",
    "sorted_assay_sizes = sorted(assay_sizes, key=lambda x:x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEICAYAAAC0+DhzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFdJJREFUeJzt3X+wZGWd3/H3RxDK8kdA+VEEiEN0yghuMrATJMUmQd3AgEkGspLAGplQmHG3hiqNpMphK1VYGgxmS60iKrUYJgwbdSSrFlPLrDhSEN1EkMEl/AxhFmfhOiMMDiiuWyj4zR/nuW57T98fc+/M7fvj/arq6u5vP+fcp0/19Gee55w+J1WFJEmDXjbqDkiSFh7DQZLUYzhIknoMB0lSj+EgSeoxHCRJPYaDtMAkeSjJWaPuh5Y3w0FLWpI7kzyb5PBR92VcksOSfCLJWJKfJPlekk+Nv15Vp1TVnSPsomQ4aOlKsgL4h0AB/3yknflVVwKrgdOBVwNvA/5spD2SJjActJRdAtwF3AisG3whyXlJHk7yfJLvJ/n3rX5Ukj9O8lySfUm+leRl7bWNSf68LfNwkgta/fDW9tcG1n9Mkr9KcvSQfv194KtVtbs6u6rqpoFldyX5zfb4uTa6+EmSv0xSLfRI8k+T3Nfa/O8kf/fAbTotd4eOugPSQXQJ8EngbuCuJMdW1VPttRuAf1lV30pyJHBSq18BjAHjX+pn0I08AP6cbiTyA+BC4L8neWNV7UmyBfjXwIda24uBb1TV3iH9ugv4YJKfAd8CHqxJzmNTVUeMP07yMeA3gO8nOQ3YBPwzYEf721uTvKmqXpjh9pEm5chBS1KS3wBeD9xcVffSfbH/9kCTnwMnJ3lNVT1bVd8dqB8HvL6qfl5V3xr/4q6q/9H+t/+LqvoS8Bjd1BDAZuC3x0cZwHuAP5yke/8J+Djwbrov9u8nWTdJ2/H3869a/3+rqn4O/FvgD6rq7qp6qao2Ay/QhZk0Z4aDlqp1wNer6pn2/Av86tTSbwHnAX+R5H8m+Qet/vvATuDrSR5PsnF8gSSXDEzjPAe8BTgKoKruBv4S+MdJ/g7wRmDrsI61L/PPVNWZwBHA1cCmJG8e1j7JqcCngQsGRiKvB64Y70vrz4nA35z5JpImF8/KqqUmySvopn4OAX7SyofTfRGvqqr/M9D25cDlwAer6sQJ6zkFuINuimgn8P+AdwDfrqqXktwHfLqq/mtrfyVdKPwAOLaq3rsffX4GeF9VfTnJLuC9VfWNts/iHuBDbbQy3v4PgCeq6uqZ/g1pfzhy0FJ0PvAScDKwqt3eTDe/f0k7lPTdSf5Gm6L5cWs/vpP3jUkyUH8JeCXdvoe9rd2ldCOHQX8IXEA3/38Tk0jygSRnJXlFkkPblNKrmXDEUpJDgS8Dnx8MhuZzwO8keWs6r0zyziSv3p8NJU3GcNBStA74b1X1RFX9YPxGNzXz7tbmPcCuJD8GfofuCx1gJfANuhHHt4HPVtWdVfUw8IlWewr4NeB/Df7RqhoDvksXIt+aon9/1db1A+AZYAPdvoTHJ7Q7gW4H+AcGjlj6SZK/VVU76PY7fBp4lm5k829mvIWkaTitJB1ASTYBu6vqP4y6L9JceCirdIC03x/8C+DU0fZEmjunlaQDIMlHgQeB36+q7426P9JcOa0kSepx5CBJ6lm0+xyOOuqoWrFixai7sTw8+mh3/6Y3jbYfi4Xba/+4vebVvffe+0xVDTvn169YtOGwYsUKduzYMepuLA9nndXd33nnKHuxeLi99o/ba14l+YuZtHNaSZLUYzhIknqmDYckJya5I8kj7fKF72/1D7fz4N/XbucNLHNlkp1JHk1yzkB9TavtnHBCs5OS3J3ksSRfSnLYgX6jkqSZm8nI4UXgiqp6M93pgDckObm99qmqWtVu2wDaaxcBpwBrgM8mOSTJIcBngHPpznlz8cB6Pt7WtZLuVACXHaD3J0mahWnDoar2jJ/rvqqeBx4Bjp9ikbXAlqp6of0YaCfdOe9PB3ZW1eNV9TNgC7C2neDs7cAfteU30504TZI0Ivu1z6GdHuBUuitrAVye5P4km9rVtKALjicHFhtrtcnqrwOeq6oXJ9QlSSMy43BI8iq60wd/oKp+DFwHvIHudMh76M4yCZAhi9cs6sP6sD7JjiQ79u4ddvVFSdKBMKNwaBdEGT+v/FcAquqpdkWrX9CdW378coljdFekGncCsHuK+jPAEe3c9YP1nqq6vqpWV9Xqo4+e9jcckqRZmsnRSqG7GPsjVfXJgfpxA80uoDvpGHSXRrwoyeFJTqI7P/536K5mtbIdmXQY3U7rre36vHcA72rLrwNumdvbkiTNxUxGDmfSXRjl7RMOW/3PSR5Icj/wNuDfAVTVQ8DNwMPA14ANbYTxIt3lGG+j26l9c2sL8CHgg0l20u2DuOHAvUVJWhpWbLx13v7WtKfPqKo/Zfh+gW1TLHM13UXTJ9a3DVuuXQHr9Il1SdJo+AtpSVKP4SBJ6jEcJEk9hoMkqcdwkCT1GA6SpB7DQZLUYzhIknoMB0lSj+EgSeoxHCRJPYaDJKnHcJCkRWA+z8gKhoMkaQjDQZLUYzhIknoMB0lSj+EgSeoxHCRJPYaDJKnHcJAk9RgOkqQew0GSFrD5/mX0OMNBktRjOEiSegwHSVKP4SBJ6jEcJEk9hoMkqcdwkCT1GA6SpB7DQZLUYzhIknqmDYckJya5I8kjSR5K8v5Wf22S7Ukea/dHtnqSXJtkZ5L7k5w2sK51rf1jSdYN1H89yQNtmWuT5GC8WUnSzMxk5PAicEVVvRk4A9iQ5GRgI3B7Va0Ebm/PAc4FVrbbeuA66MIEuAp4K3A6cNV4oLQ26weWWzP3tyZJmq1pw6Gq9lTVd9vj54FHgOOBtcDm1mwzcH57vBa4qTp3AUckOQ44B9heVfuq6llgO7Cmvfaaqvp2VRVw08C6JEkjsF/7HJKsAE4F7gaOrao90AUIcExrdjzw5MBiY602VX1sSF2SNCIzDockrwK+DHygqn48VdMhtZpFfVgf1ifZkWTH3r17p+uyJGmWZhQOSV5OFwyfr6qvtPJTbUqIdv90q48BJw4sfgKwe5r6CUPqPVV1fVWtrqrVRx999Ey6LkmahZkcrRTgBuCRqvrkwEtbgfEjjtYBtwzUL2lHLZ0B/KhNO90GnJ3kyLYj+mzgtvba80nOaH/rkoF1SZJG4NAZtDkTeA/wQJL7Wu33gGuAm5NcBjwBXNhe2wacB+wEfgpcClBV+5J8FLintftIVe1rj38XuBF4BfAn7SZJGpFpw6Gq/pTh+wUA3jGkfQEbJlnXJmDTkPoO4C3T9UWSND/8hbQkqcdwkKQFasXGW0f2tw0HSVKP4SBJ6pnJ0UqSpHk0yumkcY4cJEk9hoMkqcdwkCT1GA6SpB7DQZIWkIWwMxoMB0nSEIaDJKnHcJAk9RgOkqQew0GS1GM4SJJ6DAdJUo/hIEnqMRwkST2GgySpx3CQJPUYDpKkHsNBktRjOEiSegwHSVKP4SBJ6jEcJEk9hoMkqcdwkCT1GA6SpB7DQZLUYzhIknoMB0lSj+EgSeqZNhySbErydJIHB2ofTvL9JPe123kDr12ZZGeSR5OcM1Bf02o7k2wcqJ+U5O4kjyX5UpLDDuQblCTtv5mMHG4E1gypf6qqVrXbNoAkJwMXAae0ZT6b5JAkhwCfAc4FTgYubm0BPt7WtRJ4FrhsLm9IkjR304ZDVX0T2DfD9a0FtlTVC1X1PWAncHq77ayqx6vqZ8AWYG2SAG8H/qgtvxk4fz/fgyTpAJvLPofLk9zfpp2ObLXjgScH2oy12mT11wHPVdWLE+pDJVmfZEeSHXv37p1D1yVJU5ltOFwHvAFYBewBPtHqGdK2ZlEfqqqur6rVVbX66KOP3r8eS5Jm7NDZLFRVT40/TvI54I/b0zHgxIGmJwC72+Nh9WeAI5Ic2kYPg+0lSSMyq5FDkuMGnl4AjB/JtBW4KMnhSU4CVgLfAe4BVrYjkw6j22m9taoKuAN4V1t+HXDLbPokSTpwph05JPkicBZwVJIx4CrgrCSr6KaAdgHvA6iqh5LcDDwMvAhsqKqX2nouB24DDgE2VdVD7U98CNiS5D8CfwbccMDenSRpVqYNh6q6eEh50i/wqroauHpIfRuwbUj9cbqjmSRJC4S/kJYk9RgOkqQew0GS1GM4SJJ6DAdJUo/hIEnqMRwkST2GgySpx3CQJPUYDpKkHsNBktRjOEiSegwHSVKP4SBJ6jEcJEk9hoMkqcdwkCT1GA6SpB7DQZLUYzhIknoMB0lSj+EgSeoxHCRJPYaDJKnHcJAk9RgOkrRArNh466i78EuGgySpx3CQJPUYDpKkHsNBktRjOEiSegwHSVKP4SBJ6pk2HJJsSvJ0kgcHaq9Nsj3JY+3+yFZPkmuT7Exyf5LTBpZZ19o/lmTdQP3XkzzQlrk2SQ70m5SkhWrFxlsX1O8bxs1k5HAjsGZCbSNwe1WtBG5vzwHOBVa223rgOujCBLgKeCtwOnDVeKC0NusHlpv4tyRJ82zacKiqbwL7JpTXApvb483A+QP1m6pzF3BEkuOAc4DtVbWvqp4FtgNr2muvqapvV1UBNw2sS5KWpIU6Whg0230Ox1bVHoB2f0yrHw88OdBurNWmqo8NqQ+VZH2SHUl27N27d5Zdl6TRWOiBMOjQA7y+YfsLahb1oarqeuB6gNWrV0/aTpLm04qNt7Lrmnf+8st/8PFiNdtweCrJcVW1p00NPd3qY8CJA+1OAHa3+lkT6ne2+glD2kvSjOzPF/Nc2y2FL/2Zmm04bAXWAde0+1sG6pcn2UK38/lHLUBuAz42sBP6bODKqtqX5PkkZwB3A5cA/2WWfZK0hE38Ul5OX9SjMG04JPki3f/6j0oyRnfU0TXAzUkuA54ALmzNtwHnATuBnwKXArQQ+ChwT2v3kaoa38n9u3RHRL0C+JN2k7SMLcf/qS8004ZDVV08yUvvGNK2gA2TrGcTsGlIfQfwlun6IWnpWbHxVrY8/kMALhoIBI2ev5CWdFCM/49/8LDNiY+1cB3oo5UkLUFTzfc79bM0GQ7SMjPVF7lf+hrntJK0xA2b0pGm48hBWoT291h9aX8ZDtICN2y+XzrYDAdpRJbiKRe0dBgO0jzyy1+LheEgzQNDQYuNRytJknoMB+kgcsSgxcpwkA6gFRtv5a52riCDQYuZ4SBJ6nGHtHQAOErQUuPIQZojg0FLkSMHaZYMBS1ljhykKUx3TQJpqTIcpEkYAFrODAdJUo/7HKQBjhakjiMHSVKP4aBlzwveS32Gg5Y1A0EaznCQJPUYDpKkHsNBktRjOEiSegwHSVKP4SBJ6jEcJEk9hoMkqcdwkCT1GA6SpJ45hUOSXUkeSHJfkh2t9tok25M81u6PbPUkuTbJziT3JzltYD3rWvvHkqyb21uSJM3VgRg5vK2qVlXV6vZ8I3B7Va0Ebm/PAc4FVrbbeuA66MIEuAp4K3A6cNV4oEiSRuNgTCutBTa3x5uB8wfqN1XnLuCIJMcB5wDbq2pfVT0LbAfWHIR+SZJmaK7hUMDXk9ybZH2rHVtVewDa/TGtfjzw5MCyY602Wb0nyfokO5Ls2Lt37xy7ruXMs7FKU5vrleDOrKrdSY4Btif5v1O0zZBaTVHvF6uuB64HWL169dA2kqS5m9PIoap2t/unga/S7TN4qk0X0e6fbs3HgBMHFj8B2D1FXZI0IrMeOSR5JfCyqnq+PT4b+AiwFVgHXNPub2mLbAUuT7KFbufzj6pqT5LbgI8N7IQ+G7hytv2SJuNUkjRzc5lWOhb4apLx9Xyhqr6W5B7g5iSXAU8AF7b224DzgJ3AT4FLAapqX5KPAve0dh+pqn1z6JckaY5mHQ5V9Tjw94bUfwi8Y0i9gA2TrGsTsGm2fZEkHVj+QlrLglNK0v4xHLTkGQzS/jMcJEk9hoOWlPFRwoqNtzpikObAcJAk9cz1F9LSSEwcFey65p2OFKQDyJGDJKnHcNCi4whBOvgMB0lSj+GgRcVRgzQ/DActWB6WKo2O4aAFyTCQRstDWbUgGAbSwuLIQSNlKEgLk+EgSeoxHCRJPYaDJKnHcJAk9RgOkqQew0HzbvDHbZIWJsNBktRjOEiSegwHSVKP4SBJ6jEcJEk9hoPmhUcoSYuL4SBJ6jEcJEk9hoMkqcdw0EHnfgZp8fFKcNpvE7/sd13zzl/WBh9LWrwMB01pxcZb2fL4Dznjb7/OL31pGXFaSb80eLipQSAtbwsmHJKsSfJokp1JNo66P8uNYSBp0IKYVkpyCPAZ4J8AY8A9SbZW1cOj7dnSZRhImsqCCAfgdGBnVT0OkGQLsBYYWThM9uU5052vk7WbzTIHY92SNJVU1aj7QJJ3AWuq6r3t+XuAt1bV5RParQfWt6dvAh6d5Z88CnhmlssuB26fqbl9pub2mdqot8/rq+ro6RotlJFDhtR6qVVV1wPXz/mPJTuqavVc17NUuX2m5vaZmttnaotl+yyUHdJjwIkDz08Ado+oL5K07C2UcLgHWJnkpCSHARcBW0fcJ0lathbEtFJVvZjkcuA24BBgU1U9dBD/5JynppY4t8/U3D5Tc/tMbVFsnwWxQ1qStLAslGklSdICYjhIknqWXTh4mo6+JLuSPJDkviQ7Wu21SbYneazdHznqfs6XJJuSPJ3kwYHa0O2RzrXt83R/ktNG1/P5Mcn2+XCS77fP0H1Jzht47cq2fR5Ncs5oej0/kpyY5I4kjyR5KMn7W33RfX6WVTgMnKbjXOBk4OIkJ4+2VwvG26pq1cDx1xuB26tqJXB7e75c3AismVCbbHucC6xst/XAdfPUx1G6kf72AfhU+wytqqptAO3f10XAKW2Zz7Z/h0vVi8AVVfVm4AxgQ9sGi+7zs6zCgYHTdFTVz4Dx03Soby2wuT3eDJw/wr7Mq6r6JrBvQnmy7bEWuKk6dwFHJDlufno6GpNsn8msBbZU1QtV9T1gJ92/wyWpqvZU1Xfb4+eBR4DjWYSfn+UWDscDTw48H2u15a6Arye5t52iBODYqtoD3QceOGZkvVsYJtsefqb+2uVtamTTwDTkst0+SVYApwJ3swg/P8stHGZ0mo5l6MyqOo1uiLshyT8adYcWET9TneuANwCrgD3AJ1p9WW6fJK8Cvgx8oKp+PFXTIbUFsX2WWzh4mo4hqmp3u38a+CrdsP+p8eFtu396dD1cECbbHn6mgKp6qqpeqqpfAJ/jr6eOlt32SfJyumD4fFV9pZUX3ednuYWDp+mYIMkrk7x6/DFwNvAg3XZZ15qtA24ZTQ8XjMm2x1bgknbUyRnAj8anD5aTCfPkF9B9hqDbPhclOTzJSXQ7Xr8z3/2bL0kC3AA8UlWfHHhp0X1+FsTpM+bLCE7TsRgcC3y1+0xzKPCFqvpaknuAm5NcBjwBXDjCPs6rJF8EzgKOSjIGXAVcw/DtsQ04j25H60+BS+e9w/Nsku1zVpJVdFMiu4D3AVTVQ0luprs2y4vAhqp6aRT9nidnAu8BHkhyX6v9Hovw8+PpMyRJPcttWkmSNAOGgySpx3CQJPUYDpKkHsNBktRjOEiSegwHSVLP/wf/ukM5V7KVcQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(np.arange(len(sorted_assay_sizes)), [i[1] for i in sorted_assay_sizes])\n",
    "plt.axvline(x=53, color='red')\n",
    "plt.axvline(x=53*2, color='red')\n",
    "plt.axvline(x=53*3, color='red')\n",
    "plt.title('Assay Size')\n",
    " \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Generate Fingerprint features per assay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the inchi list to a fp list\n",
    "featurizer = dc.feat.CircularFingerprint(size=1024)\n",
    "output_fps = [featurizer._featurize(Chem.inchi.MolFromInchi(str(i))).ToBitString()\n",
    "              for i in output_inchi]\n",
    "output_fps = np.array(output_fps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp_features = []\n",
    "\n",
    "for c in range(output_matrix.shape[1]):\n",
    "    fp_features.append(output_fps[output_matrix[:, c] != -1])\n",
    "    \n",
    "fp_features = np.array(fp_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create csv files. Each assay has one csv file.\n",
    "\n",
    "for c in range(len(fp_features)):\n",
    "    cur_dict = {\n",
    "        # 'index': list(range(1, len(fp_features[c]) + 1)),\n",
    "        'index': list(range(0, len(fp_features[c]))),\n",
    "        'feature': fp_features[c].tolist()\n",
    "    }\n",
    "    cur_df = pd.DataFrame(cur_dict)\n",
    "    cur_df.to_csv(\"./temp/fp_csvs/assay_{}.csv\".format(c), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2. Collect Dissimialrity Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_dissimilarities = load(open('./temp/avg_similarities.json', 'r'))\n",
    "cpd_data = [(i,\n",
    "             np.sum(output_matrix[:,i] != -1),\n",
    "             avg_dissimilarities[i]) for i in range(len(avg_dissimilarities))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>size</th>\n",
       "      <th>diversity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3914</td>\n",
       "      <td>0.861321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>695</td>\n",
       "      <td>0.873354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9691</td>\n",
       "      <td>0.866206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>0.833961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>56</td>\n",
       "      <td>0.843337</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  size  diversity\n",
       "0      0  3914   0.861321\n",
       "1      1   695   0.873354\n",
       "2      2  9691   0.866206\n",
       "3      3    32   0.833961\n",
       "4      4    56   0.843337"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(cpd_data)\n",
    "df.columns = ['index', 'size', 'diversity']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('./temp/assay_diversity.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3. Compare rdkit similarity and customized dissimilarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_100_inchi = shuffle(output_inchi)[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_100_inchi = load(open('random_100_inchi.json', 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute the pairwise similarity of these 100 randomly sampled inchi\n",
    "similarity_matrix = np.ones((100, 100)) * -1\n",
    "\n",
    "# Convert inchi to fps\n",
    "random_100_fps = []\n",
    "for i in random_100_inchi:\n",
    "    mol = Chem.inchi.MolFromInchi(str(i))\n",
    "    fp1= Chem.rdMolDescriptors.GetMorganFingerprintAsBitVect(mol,\n",
    "          radius=2,\n",
    "          nBits=2048,\n",
    "          useChirality=False,\n",
    "          useBondTypes=True,\n",
    "          useFeatures=False\n",
    "    )\n",
    "    # Use fp1, do not use fp2!\n",
    "    fp2 = FingerprintMols.FingerprintMol(mol)\n",
    "    random_100_fps.append(fp1)\n",
    "    \n",
    "\n",
    "# Compute pairwise similarities\n",
    "for i in range(len(random_100_fps)):\n",
    "    for j in range(i + 1, len(random_100_fps)):\n",
    "        similarity_matrix[i, j] = DataStructs.TanimotoSimilarity(\n",
    "            random_100_fps[i],\n",
    "            random_100_fps[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.        ,  0.13684211,  0.10526316, ...,  0.12121212,\n",
       "         0.09649123,  0.1744186 ],\n",
       "       [-1.        , -1.        ,  0.11666667, ...,  0.11214953,\n",
       "         0.09016393,  0.17204301],\n",
       "       [-1.        , -1.        , -1.        , ...,  0.23423423,\n",
       "         0.39814815,  0.08547009],\n",
       "       ..., \n",
       "       [-1.        , -1.        , -1.        , ..., -1.        ,\n",
       "         0.14285714,  0.08737864],\n",
       "       [-1.        , -1.        , -1.        , ..., -1.        ,\n",
       "        -1.        ,  0.0862069 ],\n",
       "       [-1.        , -1.        , -1.        , ..., -1.        ,\n",
       "        -1.        , -1.        ]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the inchi list to a fp list\n",
    "featurizer = dc.feat.CircularFingerprint(size=2048)\n",
    "random_100_fps_2 = [featurizer._featurize(Chem.inchi.MolFromInchi(str(i))).ToBitString()\n",
    "                    for i in random_100_inchi]\n",
    "random_100_fps_2 = np.array(random_100_fps_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create csv files.\n",
    "cur_dict = {\n",
    "    # 'index': list(range(1, len(fp_features[c]) + 1)),\n",
    "    'index': list(range(0, len(random_100_fps_2))),\n",
    "    'feature': random_100_fps_2.tolist()\n",
    "}\n",
    "cur_df = pd.DataFrame(cur_dict)\n",
    "cur_df.to_csv(\"./temp/rand_100.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dissimilarity_matrix = np.memmap('./temp/dissimilarity_matrix_100_100_100.dat',\n",
    "                                mode='r', dtype=np.float16, shape=(100, 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "memmap([[ 0.        ,  0.86328125,  0.89453125, ...,  0.87890625,\n",
       "          0.90332031,  0.82568359],\n",
       "        [ 0.86328125,  0.        ,  0.88330078, ...,  0.88769531,\n",
       "          0.90966797,  0.828125  ],\n",
       "        [ 0.89453125,  0.88330078,  0.        , ...,  0.765625  ,\n",
       "          0.60205078,  0.91455078],\n",
       "        ..., \n",
       "        [ 0.87890625,  0.88769531,  0.765625  , ...,  0.        ,\n",
       "          0.85693359,  0.91259766],\n",
       "        [ 0.90332031,  0.90966797,  0.60205078, ...,  0.85693359,\n",
       "          0.        ,  0.91357422],\n",
       "        [ 0.82568359,  0.828125  ,  0.91455078, ...,  0.91259766,\n",
       "          0.91357422,  0.        ]], dtype=float16)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dissimilarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.        ,  1.11883681,  1.19459914, ...,  1.21923412,\n",
       "         1.2370908 ,  1.1686588 ],\n",
       "       [-0.13671875, -1.        ,  1.21689514, ...,  1.22394622,\n",
       "         1.21576116,  1.11362796],\n",
       "       [-0.10546875, -0.11669922, -1.        , ...,  1.24546095,\n",
       "         1.15464444,  1.26726143],\n",
       "       ..., \n",
       "       [-0.12109375, -0.11230469, -0.234375  , ..., -1.        ,\n",
       "         1.27644882,  1.28759766],\n",
       "       [-0.09667969, -0.09033203, -0.39794922, ..., -0.14306641,\n",
       "        -1.        ,  1.28833881],\n",
       "       [-0.17431641, -0.171875  , -0.08544922, ..., -0.08740234,\n",
       "        -0.08642578, -1.        ]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dissimilarity_matrix + similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities = []\n",
    "dissimilarities = []\n",
    "\n",
    "for i in range(100):\n",
    "    for j in range(i + 1, 100):\n",
    "        similarities.append(similarity_matrix[i, j])\n",
    "        dissimilarities.append(dissimilarity_matrix[i, j])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}