diff --git a/review/2-1-stats/calc_stats.ipynb b/review/2-1-stats/calc_stats.ipynb
new file mode 100644
index 0000000..f00fa5e
--- /dev/null
+++ b/review/2-1-stats/calc_stats.ipynb
@@ -0,0 +1,1248 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import os\n",
+    "from os.path import dirname, realpath, join\n",
+    "base_dir = dirname(dirname(os.getcwd()))\n",
+    "import itertools\n",
+    "import pandas as pd\n",
+    "from os.path import join\n",
+    "base_dir\n",
+    "\n",
+    "sys.path.insert(0, base_dir)\n",
+    "from config_path import PROSTATE_DATA_PATH, PLOTS_PATH, GENE_PATH, PROSTATE_LOG_PATH\n",
+    "from data.data_access import Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils.stats_utils_delong_xu import delong_roc_variance, delong_roc_test\n",
+    "from matplotlib import  pyplot as plt\n",
+    "from utils.stats_utils import score_ci, pvalue, pvalue_stat\n",
+    "from sklearn import metrics\n",
+    "import numpy as np "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def read_predictions():\n",
+    "    all_models_dict = {}\n",
+    "    base_dir = PROSTATE_LOG_PATH\n",
+    "    models_base_dir = join(base_dir , 'compare/onsplit_ML_test')\n",
+    "    models = ['Linear Support Vector Machine ', 'RBF Support Vector Machine ', 'L2 Logistic Regression', 'Random Forest',\n",
+    "              'Adaptive Boosting', 'Decision Tree']\n",
+    "\n",
+    "    for i, m in enumerate(models):\n",
+    "        df = pd.read_csv(join(models_base_dir, m + '_data_0_testing.csv'), sep=',', index_col=0, header=0)\n",
+    "        all_models_dict[m] = df\n",
+    "\n",
+    "    pnet_base_dir = join(base_dir , 'pnet/onsplit_average_reg_10_tanh_large_testing')\n",
+    "    df_pnet = pd.read_csv(join(pnet_base_dir, 'P-net_ALL_testing.csv'), sep=',', index_col=0, header=0)\n",
+    "    all_models_dict['P-net'] = df_pnet\n",
+    "    return all_models_dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_models_dict = read_predictions()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pnet_predictions = all_models_dict['P-net']\n",
+    "labels = pnet_predictions['y'].values.ravel()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>pred</th>\n",
+       "      <th>pred_scores</th>\n",
+       "      <th>y</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>01-087MM_BONE</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0.946647</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>01-095N1_LN</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.127979</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>08-093J1_LN</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0.990789</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10362</th>\n",
+       "      <td>1.0</td>\n",
+       "      <td>0.475796</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AAPC-IP_LG-069-Tumor-SM-3NC72</th>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.114404</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               pred  pred_scores  y\n",
+       "01-087MM_BONE                   1.0     0.946647  1\n",
+       "01-095N1_LN                     0.0     0.127979  1\n",
+       "08-093J1_LN                     1.0     0.990789  1\n",
+       "10362                           1.0     0.475796  0\n",
+       "AAPC-IP_LG-069-Tumor-SM-3NC72   0.0     0.114404  0"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_models_dict['P-net'].head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>pred</th>\n",
+       "      <th>pred_scores</th>\n",
+       "      <th>y</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>01-087MM_BONE</th>\n",
+       "      <td>1</td>\n",
+       "      <td>0.765949</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>01-095N1_LN</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0.242726</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>08-093J1_LN</th>\n",
+       "      <td>1</td>\n",
+       "      <td>0.956951</td>\n",
+       "      <td>1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10362</th>\n",
+       "      <td>1</td>\n",
+       "      <td>0.809066</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AAPC-IP_LG-069-Tumor-SM-3NC72</th>\n",
+       "      <td>0</td>\n",
+       "      <td>0.238246</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                               pred  pred_scores  y\n",
+       "01-087MM_BONE                     1     0.765949  1\n",
+       "01-095N1_LN                       0     0.242726  1\n",
+       "08-093J1_LN                       1     0.956951  1\n",
+       "10362                             1     0.809066  0\n",
+       "AAPC-IP_LG-069-Tumor-SM-3NC72     0     0.238246  0"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_models_dict['Linear Support Vector Machine '].head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "score_fun={}\n",
+    "score_fun['Accuracy'] = metrics.accuracy_score\n",
+    "score_fun['Precision'] = metrics.precision_score\n",
+    "score_fun['AUC'] = metrics.roc_auc_score\n",
+    "score_fun['F1'] = metrics.f1_score\n",
+    "score_fun['AUPR'] = metrics.average_precision_score\n",
+    "score_fun['Recall'] = metrics.recall_score\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def fdr(p_vals):\n",
+    "#https://stackoverflow.com/questions/25185205/calculating-adjusted-p-values-in-python\n",
+    "    from scipy.stats import rankdata\n",
+    "    ranked_p_values = rankdata(p_vals)\n",
+    "    fdr = p_vals * len(p_vals) / ranked_p_values\n",
+    "    fdr[fdr > 1] = 1\n",
+    "\n",
+    "    return fdr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results=[]\n",
+    "pvalue_list=[]\n",
+    "delong_results=[]\n",
+    "for i, (model_name, predictions_df) in enumerate(all_models_dict.items()):\n",
+    "    if model_name!='P-net':\n",
+    "        pred2 = predictions_df['pred_scores'].values.ravel()\n",
+    "        for func_name, func  in score_fun.items():\n",
+    "                \n",
+    "            if func_name in ['AUC', 'AUPR']:\n",
+    "                col_name= 'pred_scores'\n",
+    "            else:\n",
+    "                col_name= 'pred'\n",
+    "                \n",
+    "            pred_pnet= pnet_predictions[col_name].values.ravel()\n",
+    "            pred_model= predictions_df[col_name].values.ravel()\n",
+    "            \n",
+    "            if func_name=='AUC':\n",
+    "                pvalue_ = delong_roc_test(labels, pred_pnet, pred_model)\n",
+    "                pvalue_delong = 10**pvalue_[0][0]/2\n",
+    "                delong_results.append({'measure': 'AUC_DeLong', 'model':model_name, 'pvalue': pvalue_delong}) \n",
+    "            \n",
+    "            stat_fun= np.median\n",
+    "            p, scores1, scores2, z = pvalue_stat( labels,pred_pnet,pred_model, func,n_bootstraps=2000, two_tailed=False, seed=1234, stat_fun=np.median)\n",
+    "            med_pnet = stat_fun(scores1)\n",
+    "            med_model = stat_fun(scores2)\n",
+    "            stat_fun_diff =  med_pnet - med_model\n",
+    "            results.append({'measure': func_name, 'model':model_name, 'pvalue': p, 'model median': med_model, 'P-NET median':med_pnet, 'Median difference': stat_fun_diff }) \n",
+    "            \n",
+    "            "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results_df = pd.DataFrame(results)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results_df['pvalue_fdr_adjusted']= results_df.groupby('measure')['pvalue'].apply(fdr)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results_df = results_df.set_index(['model', 'measure'])\n",
+    "results_df = results_df.round(3)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>Median difference</th>\n",
+       "      <th>P-NET median</th>\n",
+       "      <th>model median</th>\n",
+       "      <th>pvalue</th>\n",
+       "      <th>pvalue_fdr_adjusted</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>model</th>\n",
+       "      <th>measure</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">RBF Support Vector Machine</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <td>0.021</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.860</td>\n",
+       "      <td>0.205</td>\n",
+       "      <td>0.205</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>F1</th>\n",
+       "      <td>0.054</td>\n",
+       "      <td>0.755</td>\n",
+       "      <td>0.702</td>\n",
+       "      <td>0.088</td>\n",
+       "      <td>0.177</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AUC</th>\n",
+       "      <td>0.013</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.915</td>\n",
+       "      <td>0.212</td>\n",
+       "      <td>0.212</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Recall</th>\n",
+       "      <td>0.151</td>\n",
+       "      <td>0.763</td>\n",
+       "      <td>0.612</td>\n",
+       "      <td>0.002</td>\n",
+       "      <td>0.003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Precision</th>\n",
+       "      <td>-0.074</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.824</td>\n",
+       "      <td>0.927</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Accuracy</th>\n",
+       "      <td>0.010</td>\n",
+       "      <td>0.838</td>\n",
+       "      <td>0.828</td>\n",
+       "      <td>0.372</td>\n",
+       "      <td>0.745</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">Decision Tree</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <td>0.140</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.741</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.004</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>F1</th>\n",
+       "      <td>0.035</td>\n",
+       "      <td>0.755</td>\n",
+       "      <td>0.720</td>\n",
+       "      <td>0.207</td>\n",
+       "      <td>0.248</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AUC</th>\n",
+       "      <td>0.072</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.856</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Recall</th>\n",
+       "      <td>0.147</td>\n",
+       "      <td>0.763</td>\n",
+       "      <td>0.615</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Precision</th>\n",
+       "      <td>-0.122</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.872</td>\n",
+       "      <td>0.978</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Accuracy</th>\n",
+       "      <td>-0.005</td>\n",
+       "      <td>0.838</td>\n",
+       "      <td>0.843</td>\n",
+       "      <td>0.602</td>\n",
+       "      <td>0.722</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">L2 Logistic Regression</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <td>0.068</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.813</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.004</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>F1</th>\n",
+       "      <td>0.028</td>\n",
+       "      <td>0.755</td>\n",
+       "      <td>0.727</td>\n",
+       "      <td>0.232</td>\n",
+       "      <td>0.232</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AUC</th>\n",
+       "      <td>0.045</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.883</td>\n",
+       "      <td>0.006</td>\n",
+       "      <td>0.018</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Recall</th>\n",
+       "      <td>0.091</td>\n",
+       "      <td>0.763</td>\n",
+       "      <td>0.672</td>\n",
+       "      <td>0.066</td>\n",
+       "      <td>0.066</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Precision</th>\n",
+       "      <td>-0.042</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.792</td>\n",
+       "      <td>0.826</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Accuracy</th>\n",
+       "      <td>0.005</td>\n",
+       "      <td>0.838</td>\n",
+       "      <td>0.833</td>\n",
+       "      <td>0.452</td>\n",
+       "      <td>0.679</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">Adaptive Boosting</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <td>0.050</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.831</td>\n",
+       "      <td>0.054</td>\n",
+       "      <td>0.080</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>F1</th>\n",
+       "      <td>0.050</td>\n",
+       "      <td>0.755</td>\n",
+       "      <td>0.705</td>\n",
+       "      <td>0.126</td>\n",
+       "      <td>0.188</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AUC</th>\n",
+       "      <td>0.039</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.889</td>\n",
+       "      <td>0.019</td>\n",
+       "      <td>0.038</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Recall</th>\n",
+       "      <td>0.195</td>\n",
+       "      <td>0.763</td>\n",
+       "      <td>0.568</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Precision</th>\n",
+       "      <td>-0.180</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.930</td>\n",
+       "      <td>0.998</td>\n",
+       "      <td>0.998</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Accuracy</th>\n",
+       "      <td>-0.005</td>\n",
+       "      <td>0.838</td>\n",
+       "      <td>0.843</td>\n",
+       "      <td>0.609</td>\n",
+       "      <td>0.609</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">Linear Support Vector Machine</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <td>0.024</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.857</td>\n",
+       "      <td>0.187</td>\n",
+       "      <td>0.224</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>F1</th>\n",
+       "      <td>0.060</td>\n",
+       "      <td>0.755</td>\n",
+       "      <td>0.695</td>\n",
+       "      <td>0.066</td>\n",
+       "      <td>0.399</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AUC</th>\n",
+       "      <td>0.021</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.907</td>\n",
+       "      <td>0.126</td>\n",
+       "      <td>0.151</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Recall</th>\n",
+       "      <td>0.151</td>\n",
+       "      <td>0.763</td>\n",
+       "      <td>0.612</td>\n",
+       "      <td>0.002</td>\n",
+       "      <td>0.003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Precision</th>\n",
+       "      <td>-0.057</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.807</td>\n",
+       "      <td>0.860</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Accuracy</th>\n",
+       "      <td>0.015</td>\n",
+       "      <td>0.838</td>\n",
+       "      <td>0.824</td>\n",
+       "      <td>0.298</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th rowspan=\"6\" valign=\"top\">Random Forest</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <td>0.058</td>\n",
+       "      <td>0.881</td>\n",
+       "      <td>0.823</td>\n",
+       "      <td>0.022</td>\n",
+       "      <td>0.043</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>F1</th>\n",
+       "      <td>0.064</td>\n",
+       "      <td>0.755</td>\n",
+       "      <td>0.691</td>\n",
+       "      <td>0.075</td>\n",
+       "      <td>0.225</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>AUC</th>\n",
+       "      <td>0.033</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.895</td>\n",
+       "      <td>0.049</td>\n",
+       "      <td>0.073</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Recall</th>\n",
+       "      <td>0.155</td>\n",
+       "      <td>0.763</td>\n",
+       "      <td>0.608</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.004</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Precision</th>\n",
+       "      <td>-0.054</td>\n",
+       "      <td>0.750</td>\n",
+       "      <td>0.804</td>\n",
+       "      <td>0.850</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Accuracy</th>\n",
+       "      <td>0.015</td>\n",
+       "      <td>0.838</td>\n",
+       "      <td>0.824</td>\n",
+       "      <td>0.310</td>\n",
+       "      <td>0.928</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                          Median difference  P-NET median  \\\n",
+       "model                          measure                                      \n",
+       "RBF Support Vector Machine     AUPR                   0.021         0.881   \n",
+       "                               F1                     0.054         0.755   \n",
+       "                               AUC                    0.013         0.928   \n",
+       "                               Recall                 0.151         0.763   \n",
+       "                               Precision             -0.074         0.750   \n",
+       "                               Accuracy               0.010         0.838   \n",
+       "Decision Tree                  AUPR                   0.140         0.881   \n",
+       "                               F1                     0.035         0.755   \n",
+       "                               AUC                    0.072         0.928   \n",
+       "                               Recall                 0.147         0.763   \n",
+       "                               Precision             -0.122         0.750   \n",
+       "                               Accuracy              -0.005         0.838   \n",
+       "L2 Logistic Regression         AUPR                   0.068         0.881   \n",
+       "                               F1                     0.028         0.755   \n",
+       "                               AUC                    0.045         0.928   \n",
+       "                               Recall                 0.091         0.763   \n",
+       "                               Precision             -0.042         0.750   \n",
+       "                               Accuracy               0.005         0.838   \n",
+       "Adaptive Boosting              AUPR                   0.050         0.881   \n",
+       "                               F1                     0.050         0.755   \n",
+       "                               AUC                    0.039         0.928   \n",
+       "                               Recall                 0.195         0.763   \n",
+       "                               Precision             -0.180         0.750   \n",
+       "                               Accuracy              -0.005         0.838   \n",
+       "Linear Support Vector Machine  AUPR                   0.024         0.881   \n",
+       "                               F1                     0.060         0.755   \n",
+       "                               AUC                    0.021         0.928   \n",
+       "                               Recall                 0.151         0.763   \n",
+       "                               Precision             -0.057         0.750   \n",
+       "                               Accuracy               0.015         0.838   \n",
+       "Random Forest                  AUPR                   0.058         0.881   \n",
+       "                               F1                     0.064         0.755   \n",
+       "                               AUC                    0.033         0.928   \n",
+       "                               Recall                 0.155         0.763   \n",
+       "                               Precision             -0.054         0.750   \n",
+       "                               Accuracy               0.015         0.838   \n",
+       "\n",
+       "                                          model median  pvalue  \\\n",
+       "model                          measure                           \n",
+       "RBF Support Vector Machine     AUPR              0.860   0.205   \n",
+       "                               F1                0.702   0.088   \n",
+       "                               AUC               0.915   0.212   \n",
+       "                               Recall            0.612   0.002   \n",
+       "                               Precision         0.824   0.927   \n",
+       "                               Accuracy          0.828   0.372   \n",
+       "Decision Tree                  AUPR              0.741   0.001   \n",
+       "                               F1                0.720   0.207   \n",
+       "                               AUC               0.856   0.000   \n",
+       "                               Recall            0.615   0.001   \n",
+       "                               Precision         0.872   0.978   \n",
+       "                               Accuracy          0.843   0.602   \n",
+       "L2 Logistic Regression         AUPR              0.813   0.001   \n",
+       "                               F1                0.727   0.232   \n",
+       "                               AUC               0.883   0.006   \n",
+       "                               Recall            0.672   0.066   \n",
+       "                               Precision         0.792   0.826   \n",
+       "                               Accuracy          0.833   0.452   \n",
+       "Adaptive Boosting              AUPR              0.831   0.054   \n",
+       "                               F1                0.705   0.126   \n",
+       "                               AUC               0.889   0.019   \n",
+       "                               Recall            0.568   0.000   \n",
+       "                               Precision         0.930   0.998   \n",
+       "                               Accuracy          0.843   0.609   \n",
+       "Linear Support Vector Machine  AUPR              0.857   0.187   \n",
+       "                               F1                0.695   0.066   \n",
+       "                               AUC               0.907   0.126   \n",
+       "                               Recall            0.612   0.002   \n",
+       "                               Precision         0.807   0.860   \n",
+       "                               Accuracy          0.824   0.298   \n",
+       "Random Forest                  AUPR              0.823   0.022   \n",
+       "                               F1                0.691   0.075   \n",
+       "                               AUC               0.895   0.049   \n",
+       "                               Recall            0.608   0.004   \n",
+       "                               Precision         0.804   0.850   \n",
+       "                               Accuracy          0.824   0.310   \n",
+       "\n",
+       "                                          pvalue_fdr_adjusted  \n",
+       "model                          measure                         \n",
+       "RBF Support Vector Machine     AUPR                     0.205  \n",
+       "                               F1                       0.177  \n",
+       "                               AUC                      0.212  \n",
+       "                               Recall                   0.003  \n",
+       "                               Precision                1.000  \n",
+       "                               Accuracy                 0.745  \n",
+       "Decision Tree                  AUPR                     0.004  \n",
+       "                               F1                       0.248  \n",
+       "                               AUC                      0.000  \n",
+       "                               Recall                   0.003  \n",
+       "                               Precision                1.000  \n",
+       "                               Accuracy                 0.722  \n",
+       "L2 Logistic Regression         AUPR                     0.004  \n",
+       "                               F1                       0.232  \n",
+       "                               AUC                      0.018  \n",
+       "                               Recall                   0.066  \n",
+       "                               Precision                1.000  \n",
+       "                               Accuracy                 0.679  \n",
+       "Adaptive Boosting              AUPR                     0.080  \n",
+       "                               F1                       0.188  \n",
+       "                               AUC                      0.038  \n",
+       "                               Recall                   0.000  \n",
+       "                               Precision                0.998  \n",
+       "                               Accuracy                 0.609  \n",
+       "Linear Support Vector Machine  AUPR                     0.224  \n",
+       "                               F1                       0.399  \n",
+       "                               AUC                      0.151  \n",
+       "                               Recall                   0.003  \n",
+       "                               Precision                1.000  \n",
+       "                               Accuracy                 1.000  \n",
+       "Random Forest                  AUPR                     0.043  \n",
+       "                               F1                       0.225  \n",
+       "                               AUC                      0.073  \n",
+       "                               Recall                   0.004  \n",
+       "                               Precision                1.000  \n",
+       "                               Accuracy                 0.928  "
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "results_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results_df.to_csv('model_comparison_pvalue.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th>pvalue</th>\n",
+       "      <th>pvalue_fdr_adjusted</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>model</th>\n",
+       "      <th>measure</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>RBF Support Vector Machine</th>\n",
+       "      <th>AUC_DeLong</th>\n",
+       "      <td>0.210</td>\n",
+       "      <td>0.210</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Decision Tree</th>\n",
+       "      <th>AUC_DeLong</th>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.006</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>L2 Logistic Regression</th>\n",
+       "      <th>AUC_DeLong</th>\n",
+       "      <td>0.007</td>\n",
+       "      <td>0.021</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Adaptive Boosting</th>\n",
+       "      <th>AUC_DeLong</th>\n",
+       "      <td>0.023</td>\n",
+       "      <td>0.046</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Linear Support Vector Machine</th>\n",
+       "      <th>AUC_DeLong</th>\n",
+       "      <td>0.117</td>\n",
+       "      <td>0.140</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Random Forest</th>\n",
+       "      <th>AUC_DeLong</th>\n",
+       "      <td>0.051</td>\n",
+       "      <td>0.076</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                           pvalue  pvalue_fdr_adjusted\n",
+       "model                          measure                                \n",
+       "RBF Support Vector Machine     AUC_DeLong   0.210                0.210\n",
+       "Decision Tree                  AUC_DeLong   0.001                0.006\n",
+       "L2 Logistic Regression         AUC_DeLong   0.007                0.021\n",
+       "Adaptive Boosting              AUC_DeLong   0.023                0.046\n",
+       "Linear Support Vector Machine  AUC_DeLong   0.117                0.140\n",
+       "Random Forest                  AUC_DeLong   0.051                0.076"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "results_delong_df = pd.DataFrame(delong_results)\n",
+    "results_delong_df['pvalue_fdr_adjusted']= results_delong_df.groupby('measure')['pvalue'].apply(fdr)\n",
+    "results_delong_df = results_delong_df.set_index(['model', 'measure'])\n",
+    "results_delong_df=results_delong_df.round(3)\n",
+    "results_delong_df.to_csv('model_comparison_delong_pvalue.csv')\n",
+    "results_delong_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th>measure</th>\n",
+       "      <th>AUC</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <th>Accuracy</th>\n",
+       "      <th>F1</th>\n",
+       "      <th>Precision</th>\n",
+       "      <th>Recall</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>model</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>Adaptive Boosting</th>\n",
+       "      <td>0.0190</td>\n",
+       "      <td>0.0535</td>\n",
+       "      <td>0.6090</td>\n",
+       "      <td>0.1255</td>\n",
+       "      <td>0.9975</td>\n",
+       "      <td>0.0000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Decision Tree</th>\n",
+       "      <td>0.0000</td>\n",
+       "      <td>0.0010</td>\n",
+       "      <td>0.6015</td>\n",
+       "      <td>0.2070</td>\n",
+       "      <td>0.9780</td>\n",
+       "      <td>0.0010</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>L2 Logistic Regression</th>\n",
+       "      <td>0.0060</td>\n",
+       "      <td>0.0010</td>\n",
+       "      <td>0.4525</td>\n",
+       "      <td>0.2325</td>\n",
+       "      <td>0.8255</td>\n",
+       "      <td>0.0660</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Linear Support Vector Machine</th>\n",
+       "      <td>0.1255</td>\n",
+       "      <td>0.1870</td>\n",
+       "      <td>0.2985</td>\n",
+       "      <td>0.0665</td>\n",
+       "      <td>0.8600</td>\n",
+       "      <td>0.0020</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>RBF Support Vector Machine</th>\n",
+       "      <td>0.2115</td>\n",
+       "      <td>0.2050</td>\n",
+       "      <td>0.3725</td>\n",
+       "      <td>0.0885</td>\n",
+       "      <td>0.9270</td>\n",
+       "      <td>0.0015</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Random Forest</th>\n",
+       "      <td>0.0485</td>\n",
+       "      <td>0.0215</td>\n",
+       "      <td>0.3095</td>\n",
+       "      <td>0.0750</td>\n",
+       "      <td>0.8495</td>\n",
+       "      <td>0.0035</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "measure                            AUC    AUPR  Accuracy      F1  Precision  \\\n",
+       "model                                                                         \n",
+       "Adaptive Boosting               0.0190  0.0535    0.6090  0.1255     0.9975   \n",
+       "Decision Tree                   0.0000  0.0010    0.6015  0.2070     0.9780   \n",
+       "L2 Logistic Regression          0.0060  0.0010    0.4525  0.2325     0.8255   \n",
+       "Linear Support Vector Machine   0.1255  0.1870    0.2985  0.0665     0.8600   \n",
+       "RBF Support Vector Machine      0.2115  0.2050    0.3725  0.0885     0.9270   \n",
+       "Random Forest                   0.0485  0.0215    0.3095  0.0750     0.8495   \n",
+       "\n",
+       "measure                         Recall  \n",
+       "model                                   \n",
+       "Adaptive Boosting               0.0000  \n",
+       "Decision Tree                   0.0010  \n",
+       "L2 Logistic Regression          0.0660  \n",
+       "Linear Support Vector Machine   0.0020  \n",
+       "RBF Support Vector Machine      0.0015  \n",
+       "Random Forest                   0.0035  "
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "df = pd.DataFrame(results)\n",
+    "df_cross = pd.crosstab( df['model'], df['measure'], values=df.pvalue, aggfunc='first') \n",
+    "\n",
+    "df_cross"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_cross_fdr_adjusted = df_cross.copy()\n",
+    "for c in df_cross_fdr_adjusted.columns:\n",
+    "    df_cross_fdr_adjusted[c] = fdr(df_cross_fdr_adjusted[c])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th>measure</th>\n",
+       "      <th>AUC</th>\n",
+       "      <th>AUPR</th>\n",
+       "      <th>Accuracy</th>\n",
+       "      <th>F1</th>\n",
+       "      <th>Precision</th>\n",
+       "      <th>Recall</th>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>model</th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "      <th></th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>Adaptive Boosting</th>\n",
+       "      <td>0.038</td>\n",
+       "      <td>0.080</td>\n",
+       "      <td>0.609</td>\n",
+       "      <td>0.188</td>\n",
+       "      <td>0.998</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Decision Tree</th>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.722</td>\n",
+       "      <td>0.248</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>L2 Logistic Regression</th>\n",
+       "      <td>0.018</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.679</td>\n",
+       "      <td>0.232</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.066</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Linear Support Vector Machine</th>\n",
+       "      <td>0.151</td>\n",
+       "      <td>0.224</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.399</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>RBF Support Vector Machine</th>\n",
+       "      <td>0.212</td>\n",
+       "      <td>0.205</td>\n",
+       "      <td>0.745</td>\n",
+       "      <td>0.177</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.003</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>Random Forest</th>\n",
+       "      <td>0.073</td>\n",
+       "      <td>0.043</td>\n",
+       "      <td>0.928</td>\n",
+       "      <td>0.225</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.004</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "measure                           AUC   AUPR  Accuracy     F1  Precision  \\\n",
+       "model                                                                      \n",
+       "Adaptive Boosting               0.038  0.080     0.609  0.188      0.998   \n",
+       "Decision Tree                   0.000  0.004     0.722  0.248      1.000   \n",
+       "L2 Logistic Regression          0.018  0.004     0.679  0.232      1.000   \n",
+       "Linear Support Vector Machine   0.151  0.224     1.000  0.399      1.000   \n",
+       "RBF Support Vector Machine      0.212  0.205     0.745  0.177      1.000   \n",
+       "Random Forest                   0.073  0.043     0.928  0.225      1.000   \n",
+       "\n",
+       "measure                         Recall  \n",
+       "model                                   \n",
+       "Adaptive Boosting                0.000  \n",
+       "Decision Tree                    0.003  \n",
+       "L2 Logistic Regression           0.066  \n",
+       "Linear Support Vector Machine    0.003  \n",
+       "RBF Support Vector Machine       0.003  \n",
+       "Random Forest                    0.004  "
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# df_cross_fdr_adjusted.applymap('{:.3f}'.format)\n",
+    "df_cross_fdr_adjusted = df_cross_fdr_adjusted.round(3)\n",
+    "df_cross_fdr_adjusted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_cross_fdr_adjusted.to_csv('df_cross_fdr_adjusted.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:min_env]",
+   "language": "python",
+   "name": "conda-env-min_env-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}