notebooks/benchmark/summary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d74d232e-7eec-4fce-ba80-9618fecd7adb",
   "metadata": {},
   "source": [
    "### First run the hyperparameter selection and benchmark"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "85be2405-aab3-4e1b-8173-0e9d05c20e57",
   "metadata": {
    "tags": []
   },
   "source": [
    "From under the directory:\n",
    "```\n",
    "tphenotype/notebooks/benchmark\n",
    "```\n",
    "Execute the `bash` script:\n",
    "```shell\n",
    "bash run_experiment.sh\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2de58c53-6b71-4697-99f2-5928b9701a50",
   "metadata": {},
   "source": [
    "### Examine benchmark results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a78bc0b1-c175-48f1-8e2b-588096cd1382",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from IPython.core.display import HTML\n",
    "from analysis import find_best_method\n",
    "from benchmarks import prepare_benchmark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2d47159b-4d9e-4370-b349-2b6ce8af374e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "metrics = [\"ROC\", \"PRC\", \"Silhouette_auc\", \"Hroc\", \"Hprc\"]\n",
    "synth_metrics = [\"ROC\", \"PRC\", \"PURITY\", \"RAND\", \"MI\", \"Silhouette_auc\", \"Hroc\", \"Hprc\"]\n",
    "\n",
    "result_dir = \"benchmark_results\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2970fcf9-d770-4bc8-a469-77d06c0e51b6",
   "metadata": {},
   "source": [
    "- KM-Laplace is the KM-$\\mathcal{L}$ baseline in the paper\n",
    "- T-Phenotype (joint) is the ablation T-phenotype (J)\n",
    "- MI = NMI (normalized mutual information)\n",
    "- Silhouette_auc = AUCSIL in the paper\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "19479e23-c18b-4ef9-90db-9c57fabcf7c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "table_order = [\n",
    "    \"KM-E2P(y)\",\n",
    "    \"KM-E2P(z)\",\n",
    "    \"KM-DTW-D\",\n",
    "    \"KM-Laplace\",\n",
    "    \"SEQ2SEQ\",\n",
    "    \"AC-TPC\",\n",
    "    \"T-Phenotype(joint)\",\n",
    "    \"T-Phenotype\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b9c2829-86ed-48c7-8229-4dc97cabb67b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for dataname in [\"Synth\", \"ICU\", \"ADNI\"]:\n",
    "    results = pd.read_csv(f\"{result_dir}/{dataname}_benchmark_complete.csv\", index_col=0)\n",
    "    results = results.reset_index(drop=True)\n",
    "    r = results.set_index(\"method\")\n",
    "    if dataname == \"Synth\":\n",
    "        r = r.loc[table_order + [\"Spectral-DTW-D\"]]  # KM + spectral clustering is possible on the synthetic dataset\n",
    "    else:\n",
    "        r = r.loc[table_order]\n",
    "    if dataname == \"Synth\":\n",
    "        m = synth_metrics\n",
    "    else:\n",
    "        m = metrics\n",
    "    ttest = find_best_method(r.reset_index(), m)\n",
    "    print(f\"dataset: {dataname}\")\n",
    "    print(\"performance\")\n",
    "    display(HTML(r[m].to_html()))\n",
    "    # p-value of two-sample test (of equal mean)\n",
    "    print(\"p-value\")\n",
    "    display(HTML(ttest.to_html()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c2dcce9-644d-4af4-b419-ddb09d606652",
   "metadata": {},
   "source": [
    "### Check hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e8e60ee-b54b-4519-8c77-6b5bfccbc5dc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def print_hparams(model, config, loss_weights):\n",
    "    if model.__name__ == \"KME2P\":\n",
    "        hparams = [\"num_layers\", \"hidden_size\", \"latent_size\"]\n",
    "        print(f'model: KME2P{config[\"latent_space\"]}')\n",
    "        for k in hparams:\n",
    "            print(f\"{k}={config[k]}\")\n",
    "        print()\n",
    "    elif model.__name__ == \"Predictor\":\n",
    "        print(f\"model: T-Phenotype\")\n",
    "        print(\"encoder parameters:\")\n",
    "        for k in [\"pole_separation\", \"max_degree\"]:\n",
    "            print(f'{k}={config[\"encoder_config\"][k]}')\n",
    "        for k in [\"pole\", \"real\"]:\n",
    "            print(f\"loss coeff {k}={loss_weights[k]}\")\n",
    "\n",
    "        print(\"predictor parameters:\")\n",
    "        for k in [\"num_layer\", \"hidden_size\"]:\n",
    "            print(f\"{k}={config[k]}\")\n",
    "        print()\n",
    "\n",
    "\n",
    "for dataname in [\"Synth\", \"ICU\", \"ADNI\"]:\n",
    "    splits, setup_list = prepare_benchmark(dataname)\n",
    "    print(\"hyperparameters\")\n",
    "\n",
    "    for model, config, loss_weights in setup_list:\n",
    "        print_hparams(model, config, loss_weights)\n",
    "    print()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ae9cdcd8-832d-4c0d-ba2f-719a272a0c1a",
   "metadata": {},
   "source": [
    "Hyperparameters of T-Phenotype\n",
    "\n",
    "Laplace encoder\n",
    "- pole = $\\alpha$\n",
    "- real = $\\alpha_1$\n",
    "- pole_separation = $\\delta_{pole}$\n",
    "- max_degree = $d$"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "review_py38_tphenotype",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}