medical.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bfaf9e81-c095-4d66-812e-3459dbc93c65",
   "metadata": {},
   "source": [
    "# Experiments on real-world data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e8602d8b-cb7c-488b-9787-1e624204f8f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from utils.train_medical import run_medical_experiments\n",
    "from utils.results import (\n",
    "    get_joint_medical_coverages, \n",
    "    get_medical_interval_widths, \n",
    "    load_medical_results, \n",
    "    get_uncorrected_medical_results\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9455748f-ff7b-4fa6-8c02-45dfd5509810",
   "metadata": {},
   "source": [
    "To obtain the results as presented in the paper, run the following three sections."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25f26614-5553-437b-9af5-2618e9aec6f1",
   "metadata": {},
   "source": [
    "## MIMIC-III dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "be909832-aa81-45cc-a2c1-af23c4bf1a2a",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 182.88025174822127\n",
      "Epoch: 50\tTrain loss: 58.05814906529018\n",
      "Epoch: 100\tTrain loss: 59.47126906258719\n",
      "Epoch: 150\tTrain loss: 72.10973262786865\n",
      "Epoch: 200\tTrain loss: 56.751835686819895\n",
      "Epoch: 250\tTrain loss: 58.04673767089844\n",
      "Epoch: 300\tTrain loss: 57.50358772277832\n",
      "Epoch: 350\tTrain loss: 57.81450939178467\n",
      "Epoch: 400\tTrain loss: 58.17083168029785\n",
      "Epoch: 450\tTrain loss: 58.1093373979841\n",
      "Epoch: 500\tTrain loss: 63.71995939527239\n",
      "Epoch: 550\tTrain loss: 58.66719668252127\n",
      "Epoch: 600\tTrain loss: 57.422222818647114\n",
      "Epoch: 650\tTrain loss: 57.164130619594026\n",
      "Epoch: 700\tTrain loss: 59.20651762826102\n",
      "Epoch: 750\tTrain loss: 60.78845092228481\n",
      "Epoch: 800\tTrain loss: 56.57934761047363\n",
      "Epoch: 850\tTrain loss: 57.23169136047363\n",
      "Epoch: 900\tTrain loss: 72.69065284729004\n",
      "Epoch: 950\tTrain loss: 56.816522870744976\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 171.21442031860352\n",
      "Epoch: 50\tTrain loss: 53.18712724958147\n",
      "Epoch: 100\tTrain loss: 30.109013898032053\n",
      "Epoch: 150\tTrain loss: 27.773639610835485\n",
      "Epoch: 200\tTrain loss: 26.425423009055002\n",
      "Epoch: 250\tTrain loss: 23.787845747811453\n",
      "Epoch: 300\tTrain loss: 24.6605578150068\n",
      "Epoch: 350\tTrain loss: 11.052408014025007\n",
      "Epoch: 400\tTrain loss: 9.12917675290789\n",
      "Epoch: 450\tTrain loss: 8.627567734037127\n",
      "Epoch: 500\tTrain loss: 7.728996889931815\n",
      "Epoch: 550\tTrain loss: 7.386963742119925\n",
      "Epoch: 600\tTrain loss: 6.706268276487078\n",
      "Epoch: 650\tTrain loss: 6.631469437054226\n",
      "Epoch: 700\tTrain loss: 13.494191544396537\n",
      "Epoch: 750\tTrain loss: 6.6220981904438565\n",
      "Epoch: 800\tTrain loss: 6.398435592651367\n",
      "Epoch: 850\tTrain loss: 5.5426967825208395\n",
      "Epoch: 900\tTrain loss: 5.4135540042604715\n",
      "Epoch: 950\tTrain loss: 4.883854406220572\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 161.31568472726005\n",
      "Epoch: 50\tTrain loss: 43.76956231253488\n",
      "Epoch: 100\tTrain loss: 44.33925451551165\n",
      "Epoch: 150\tTrain loss: 46.98996775490897\n",
      "Epoch: 200\tTrain loss: 43.808440753391814\n",
      "Epoch: 250\tTrain loss: 46.4765408379691\n",
      "Epoch: 300\tTrain loss: 43.66721085139683\n",
      "Epoch: 350\tTrain loss: 44.962336676461355\n",
      "Epoch: 400\tTrain loss: 44.33309950147356\n",
      "Epoch: 450\tTrain loss: 43.470497131347656\n",
      "Epoch: 500\tTrain loss: 44.061091150556294\n",
      "Epoch: 550\tTrain loss: 43.79562882014683\n",
      "Epoch: 600\tTrain loss: 44.08551025390625\n",
      "Epoch: 650\tTrain loss: 44.55317415509905\n",
      "Epoch: 700\tTrain loss: 44.30813544137137\n",
      "Epoch: 750\tTrain loss: 43.851208550589426\n",
      "Epoch: 800\tTrain loss: 43.167816570826936\n",
      "Epoch: 850\tTrain loss: 45.145528657095774\n",
      "Epoch: 900\tTrain loss: 44.22755690983364\n",
      "Epoch: 950\tTrain loss: 44.57049969264439\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 167.4080935886928\n",
      "Epoch: 50\tTrain loss: 55.33944034576416\n",
      "Epoch: 100\tTrain loss: 54.8204071862357\n",
      "Epoch: 150\tTrain loss: 54.86719867161342\n",
      "Epoch: 200\tTrain loss: 53.91511058807373\n",
      "Epoch: 250\tTrain loss: 58.239756039210725\n",
      "Epoch: 300\tTrain loss: 54.01619529724121\n",
      "Epoch: 350\tTrain loss: 56.63574995313372\n",
      "Epoch: 400\tTrain loss: 56.24246065957205\n",
      "Epoch: 450\tTrain loss: 55.71378231048584\n",
      "Epoch: 500\tTrain loss: 56.11356054033552\n",
      "Epoch: 550\tTrain loss: 54.15473147800991\n",
      "Epoch: 600\tTrain loss: 55.03341129847935\n",
      "Epoch: 650\tTrain loss: 54.809915270124165\n",
      "Epoch: 700\tTrain loss: 54.03415012359619\n",
      "Epoch: 750\tTrain loss: 54.1989677974156\n",
      "Epoch: 800\tTrain loss: 54.06414672306606\n",
      "Epoch: 850\tTrain loss: 54.79771041870117\n",
      "Epoch: 900\tTrain loss: 54.39642483847482\n",
      "Epoch: 950\tTrain loss: 54.86717428479876\n",
      "Training AdaptiveCFRNN\n",
      "Epoch: 0\tTrain loss: 174.52110726492745\n",
      "Epoch: 50\tTrain loss: 58.97297831944057\n",
      "Epoch: 100\tTrain loss: 56.533665248325896\n",
      "Epoch: 150\tTrain loss: 57.02759987967355\n",
      "Epoch: 200\tTrain loss: 56.44809791019985\n",
      "Epoch: 250\tTrain loss: 57.590281758989605\n",
      "Epoch: 300\tTrain loss: 58.457373891557964\n",
      "Epoch: 350\tTrain loss: 58.118039403642925\n",
      "Epoch: 400\tTrain loss: 57.71695014408657\n",
      "Epoch: 450\tTrain loss: 57.134544372558594\n",
      "Epoch: 500\tTrain loss: 57.23036316462925\n",
      "Epoch: 550\tTrain loss: 57.07126726422991\n",
      "Epoch: 600\tTrain loss: 56.92824404580252\n",
      "Epoch: 650\tTrain loss: 58.97183472769601\n",
      "Epoch: 700\tTrain loss: 57.560197830200195\n",
      "Epoch: 750\tTrain loss: 58.299515860421316\n",
      "Epoch: 800\tTrain loss: 56.730498858860564\n",
      "Epoch: 850\tTrain loss: 59.32658059256418\n",
      "Epoch: 900\tTrain loss: 57.588875089372905\n",
      "Epoch: 950\tTrain loss: 58.78564780099051\n",
      "Epoch: 0\tNormalisation loss: 2.2157968282699585\n",
      "Epoch: 100\tNormalisation loss: 1.1432762231145586\n",
      "Epoch: 200\tNormalisation loss: 1.1708456448146276\n",
      "Epoch: 300\tNormalisation loss: 1.0009532954011644\n",
      "Epoch: 400\tNormalisation loss: 0.9751043234552655\n",
      "Training AdaptiveCFRNN\n",
      "Epoch: 0\tTrain loss: 169.3331958225795\n",
      "Epoch: 50\tTrain loss: 53.623822757175994\n",
      "Epoch: 100\tTrain loss: 54.098903111049104\n",
      "Epoch: 150\tTrain loss: 30.993800367627824\n",
      "Epoch: 200\tTrain loss: 14.922206265585762\n",
      "Epoch: 250\tTrain loss: 11.840005159378052\n",
      "Epoch: 300\tTrain loss: 10.177919626235962\n",
      "Epoch: 350\tTrain loss: 9.538444859640938\n",
      "Epoch: 400\tTrain loss: 10.142385959625244\n",
      "Epoch: 450\tTrain loss: 7.605991091047015\n",
      "Epoch: 500\tTrain loss: 7.602594682148525\n",
      "Epoch: 550\tTrain loss: 8.060231515339442\n",
      "Epoch: 600\tTrain loss: 6.9587070260729105\n",
      "Epoch: 650\tTrain loss: 6.591307861464364\n",
      "Epoch: 700\tTrain loss: 6.3291391134262085\n",
      "Epoch: 750\tTrain loss: 9.620746612548828\n",
      "Epoch: 800\tTrain loss: 7.57138899394444\n",
      "Epoch: 850\tTrain loss: 7.593100956508091\n",
      "Epoch: 900\tTrain loss: 6.738853284290859\n",
      "Epoch: 950\tTrain loss: 7.068404708589826\n",
      "Epoch: 0\tNormalisation loss: 1.4193450723375594\n",
      "Epoch: 100\tNormalisation loss: nan\n",
      "Epoch: 200\tNormalisation loss: nan\n",
      "Epoch: 300\tNormalisation loss: nan\n",
      "Epoch: 400\tNormalisation loss: nan\n",
      "Training AdaptiveCFRNN\n",
      "Epoch: 0\tTrain loss: 160.51970509120397\n",
      "Epoch: 50\tTrain loss: 43.5519779750279\n",
      "Epoch: 100\tTrain loss: 44.1369252886091\n",
      "Epoch: 150\tTrain loss: 44.0650269644601\n",
      "Epoch: 200\tTrain loss: 43.46693107060024\n",
      "Epoch: 250\tTrain loss: 44.10111222948347\n",
      "Epoch: 300\tTrain loss: 45.06156962258475\n",
      "Epoch: 350\tTrain loss: 44.34012222290039\n",
      "Epoch: 400\tTrain loss: 47.14717946733747\n",
      "Epoch: 450\tTrain loss: 43.62054334368025\n",
      "Epoch: 500\tTrain loss: 44.080441338675364\n",
      "Epoch: 550\tTrain loss: 45.80099991389683\n",
      "Epoch: 600\tTrain loss: 45.58136803763254\n",
      "Epoch: 650\tTrain loss: 47.48395456586565\n",
      "Epoch: 700\tTrain loss: 44.68773678370884\n",
      "Epoch: 750\tTrain loss: 43.26656368800572\n",
      "Epoch: 800\tTrain loss: 47.00738212040493\n",
      "Epoch: 850\tTrain loss: 45.127959796360564\n",
      "Epoch: 900\tTrain loss: 49.08366176060268\n",
      "Epoch: 950\tTrain loss: 44.30769348144531\n",
      "Epoch: 0\tNormalisation loss: 2.3940626978874207\n",
      "Epoch: 100\tNormalisation loss: 1.1964898705482483\n",
      "Epoch: 200\tNormalisation loss: 1.216278076171875\n",
      "Epoch: 300\tNormalisation loss: 1.2241813710757665\n",
      "Epoch: 400\tNormalisation loss: 1.2418678530624934\n",
      "Training AdaptiveCFRNN\n",
      "Epoch: 0\tTrain loss: 168.0062037876674\n",
      "Epoch: 50\tTrain loss: 54.721711839948384\n",
      "Epoch: 100\tTrain loss: 54.56816564287458\n",
      "Epoch: 150\tTrain loss: 57.52073778424944\n",
      "Epoch: 200\tTrain loss: 56.57904965536935\n",
      "Epoch: 250\tTrain loss: 55.211440903799875\n",
      "Epoch: 300\tTrain loss: 54.494044167654856\n",
      "Epoch: 350\tTrain loss: 54.75975513458252\n",
      "Epoch: 400\tTrain loss: 58.3364189692906\n",
      "Epoch: 450\tTrain loss: 54.72922570364816\n",
      "Epoch: 500\tTrain loss: 54.283146721976145\n",
      "Epoch: 550\tTrain loss: 54.441612243652344\n",
      "Epoch: 600\tTrain loss: 53.989832741873606\n",
      "Epoch: 650\tTrain loss: 54.017670495169504\n",
      "Epoch: 700\tTrain loss: 54.182746342250276\n",
      "Epoch: 750\tTrain loss: 54.081841877528596\n",
      "Epoch: 800\tTrain loss: 55.55426270621164\n",
      "Epoch: 850\tTrain loss: 53.617226464407786\n",
      "Epoch: 900\tTrain loss: 54.62874494280134\n",
      "Epoch: 950\tTrain loss: 53.89653451102121\n",
      "Epoch: 0\tNormalisation loss: 2.2311201776776994\n",
      "Epoch: 100\tNormalisation loss: 1.3098468354770116\n",
      "Epoch: 200\tNormalisation loss: 1.3243409395217896\n",
      "Epoch: 300\tNormalisation loss: 1.301912248134613\n",
      "Epoch: 400\tNormalisation loss: 1.2302133653845106\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 1.6887\n",
      "Epoch:  1 | train loss: 1.2587\n",
      "Epoch:  2 | train loss: 1.3681\n",
      "Epoch:  3 | train loss: 1.3164\n",
      "Epoch:  4 | train loss: 1.2779\n",
      "Epoch:  5 | train loss: 1.2811\n",
      "Epoch:  6 | train loss: 1.4328\n",
      "Epoch:  7 | train loss: 1.1503\n",
      "Epoch:  8 | train loss: 1.3089\n",
      "Epoch:  9 | train loss: 1.5828\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 1.3679\n",
      "Epoch:  1 | train loss: 1.6138\n",
      "Epoch:  2 | train loss: 1.3573\n",
      "Epoch:  3 | train loss: 1.2819\n",
      "Epoch:  4 | train loss: 1.5239\n",
      "Epoch:  5 | train loss: 1.2669\n",
      "Epoch:  6 | train loss: 1.4653\n",
      "Epoch:  7 | train loss: 1.8833\n",
      "Epoch:  8 | train loss: 1.2065\n",
      "Epoch:  9 | train loss: 1.1128\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 1.6280\n",
      "Epoch:  1 | train loss: 1.2658\n",
      "Epoch:  2 | train loss: 1.4047\n",
      "Epoch:  3 | train loss: 1.2760\n",
      "Epoch:  4 | train loss: 1.3099\n",
      "Epoch:  5 | train loss: 1.3406\n",
      "Epoch:  6 | train loss: 1.5198\n",
      "Epoch:  7 | train loss: 1.7573\n",
      "Epoch:  8 | train loss: 1.3832\n",
      "Epoch:  9 | train loss: 1.2594\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 1.4005\n",
      "Epoch:  1 | train loss: 1.7694\n",
      "Epoch:  2 | train loss: 1.1601\n",
      "Epoch:  3 | train loss: 1.2333\n",
      "Epoch:  4 | train loss: 1.6755\n",
      "Epoch:  5 | train loss: 1.8015\n",
      "Epoch:  6 | train loss: 1.9593\n",
      "Epoch:  7 | train loss: 0.6193\n",
      "Epoch:  8 | train loss: 0.5742\n",
      "Epoch:  9 | train loss: 1.0167\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 34.9303\n",
      "Epoch:  1 | train loss: 60.7553\n",
      "Epoch:  2 | train loss: 28.1206\n",
      "Epoch:  3 | train loss: 21.1676\n",
      "Epoch:  4 | train loss: 56.3230\n",
      "Epoch:  5 | train loss: 29.0300\n",
      "Epoch:  6 | train loss: 105.8685\n",
      "Epoch:  7 | train loss: 9.8216\n",
      "Epoch:  8 | train loss: 20.7863\n",
      "Epoch:  9 | train loss: 17.6842\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 64.1097\n",
      "Epoch:  1 | train loss: 38.6666\n",
      "Epoch:  2 | train loss: 32.4718\n",
      "Epoch:  3 | train loss: 36.8352\n",
      "Epoch:  4 | train loss: 29.6416\n",
      "Epoch:  5 | train loss: 34.3819\n",
      "Epoch:  6 | train loss: 22.2121\n",
      "Epoch:  7 | train loss: 30.4530\n",
      "Epoch:  8 | train loss: 36.6793\n",
      "Epoch:  9 | train loss: 42.2997\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 40.7150\n",
      "Epoch:  1 | train loss: 30.1550\n",
      "Epoch:  2 | train loss: 32.4109\n",
      "Epoch:  3 | train loss: 68.9921\n",
      "Epoch:  4 | train loss: 30.7421\n",
      "Epoch:  5 | train loss: 33.8208\n",
      "Epoch:  6 | train loss: 41.6718\n",
      "Epoch:  7 | train loss: 17.0412\n",
      "Epoch:  8 | train loss: 18.7354\n",
      "Epoch:  9 | train loss: 24.7088\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 30.5944\n",
      "Epoch:  1 | train loss: 59.0719\n",
      "Epoch:  2 | train loss: 56.5863\n",
      "Epoch:  3 | train loss: 50.3570\n",
      "Epoch:  4 | train loss: 27.6397\n",
      "Epoch:  5 | train loss: 41.9052\n",
      "Epoch:  6 | train loss: 33.9332\n",
      "Epoch:  7 | train loss: 19.8412\n",
      "Epoch:  8 | train loss: 15.0589\n",
      "Epoch:  9 | train loss: 10.7664\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'AdaptiveCFRNN', 'QRNN', 'DPRNN']:\n",
    "    for seed in range(5):\n",
    "        run_medical_experiments(dataset='mimic', \n",
    "                                baseline=baseline,\n",
    "                                save_model=True, \n",
    "                                save_results=True,\n",
    "                                seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b8e8300b-92b3-45f9-8f17-4f69460c6d7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CFRNN\n",
      "94.0 \\(\\pm\\) 1.2\\%\n",
      "\n",
      "AdaptiveCFRNN\n",
      "75.0 \\(\\pm\\) 37.5\\%\n",
      "\n",
      "QRNN\n",
      "89.3 \\(\\pm\\) 1.2\\%\n",
      "\n",
      "DPRNN\n",
      "40.2 \\(\\pm\\) 13.9\\%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'AdaptiveCFRNN', 'QRNN', 'DPRNN']:\n",
    "    print(baseline)\n",
    "    coverages_mean, coverages_std = get_joint_medical_coverages(baseline, 'mimic', seeds=range(5))\n",
    "    \n",
    "    print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fe05b86b-5209-4afc-a87c-51a231698b6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CFRNN\n",
      "20.59477424621582\n",
      "3.076612983230133\n",
      "\n",
      "DPRNN\n",
      "3.594958412647247\n",
      "0.8972864178894718\n",
      "\n",
      "QRNN\n",
      "16.159785747528076\n",
      "3.921973974254315\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'DPRNN', 'QRNN']:\n",
    "    print(baseline)\n",
    "    widths_mean, widths_std = get_medical_interval_widths(baseline, 'mimic', seeds=range(5))\n",
    "    \n",
    "    print(widths_mean)\n",
    "    print(widths_std)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "084ed32a-2470-415c-aa6d-38f4c1d533fb",
   "metadata": {},
   "source": [
    "## EEG dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "646b7d52-f305-4f40-8319-2b0272447e96",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 176.828530839346\n",
      "Epoch: 50\tTrain loss: 72.87019629617339\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 178.46980163426076\n",
      "Epoch: 50\tTrain loss: 69.71923409619377\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 171.19047687123123\n",
      "Epoch: 50\tTrain loss: 77.84114200629077\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 172.37090971863387\n",
      "Epoch: 50\tTrain loss: 139.98008124342243\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 172.20736364716464\n",
      "Epoch: 50\tTrain loss: 118.57290893850974\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 2.8654\n",
      "Epoch:  1 | train loss: 2.9459\n",
      "Epoch:  2 | train loss: 2.0752\n",
      "Epoch:  3 | train loss: 1.5364\n",
      "Epoch:  4 | train loss: 1.5655\n",
      "Epoch:  5 | train loss: 1.7958\n",
      "Epoch:  6 | train loss: 1.8150\n",
      "Epoch:  7 | train loss: 1.4566\n",
      "Epoch:  8 | train loss: 1.5117\n",
      "Epoch:  9 | train loss: 1.2796\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 2.8155\n",
      "Epoch:  1 | train loss: 3.4734\n",
      "Epoch:  2 | train loss: 3.2736\n",
      "Epoch:  3 | train loss: 3.0898\n",
      "Epoch:  4 | train loss: 3.3527\n",
      "Epoch:  5 | train loss: 1.7739\n",
      "Epoch:  6 | train loss: 1.3432\n",
      "Epoch:  7 | train loss: 1.3760\n",
      "Epoch:  8 | train loss: 1.3924\n",
      "Epoch:  9 | train loss: 1.3265\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 2.6516\n",
      "Epoch:  1 | train loss: 3.0988\n",
      "Epoch:  2 | train loss: 4.3180\n",
      "Epoch:  3 | train loss: 1.5997\n",
      "Epoch:  4 | train loss: 1.4908\n",
      "Epoch:  5 | train loss: 1.4368\n",
      "Epoch:  6 | train loss: 1.2816\n",
      "Epoch:  7 | train loss: 1.3750\n",
      "Epoch:  8 | train loss: 1.2820\n",
      "Epoch:  9 | train loss: 1.4296\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 3.4488\n",
      "Epoch:  1 | train loss: 1.8199\n",
      "Epoch:  2 | train loss: 1.5679\n",
      "Epoch:  3 | train loss: 3.2401\n",
      "Epoch:  4 | train loss: 1.6197\n",
      "Epoch:  5 | train loss: 1.2852\n",
      "Epoch:  6 | train loss: 1.4772\n",
      "Epoch:  7 | train loss: 1.3974\n",
      "Epoch:  8 | train loss: 1.4514\n",
      "Epoch:  9 | train loss: 1.4631\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 3.1833\n",
      "Epoch:  1 | train loss: 3.1627\n",
      "Epoch:  2 | train loss: 4.4298\n",
      "Epoch:  3 | train loss: 2.1021\n",
      "Epoch:  4 | train loss: 1.6731\n",
      "Epoch:  5 | train loss: 1.7194\n",
      "Epoch:  6 | train loss: 1.4263\n",
      "Epoch:  7 | train loss: 1.3315\n",
      "Epoch:  8 | train loss: 1.3809\n",
      "Epoch:  9 | train loss: 1.3413\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 79.3815\n",
      "Epoch:  1 | train loss: 79.5354\n",
      "Epoch:  2 | train loss: 60.0318\n",
      "Epoch:  3 | train loss: 171.7907\n",
      "Epoch:  4 | train loss: 67.6325\n",
      "Epoch:  5 | train loss: 67.6401\n",
      "Epoch:  6 | train loss: 78.1829\n",
      "Epoch:  7 | train loss: 262.1328\n",
      "Epoch:  8 | train loss: 96.9117\n",
      "Epoch:  9 | train loss: 60.4469\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 218.6790\n",
      "Epoch:  1 | train loss: 165.4773\n",
      "Epoch:  2 | train loss: 120.4259\n",
      "Epoch:  3 | train loss: 55.2500\n",
      "Epoch:  4 | train loss: 61.8862\n",
      "Epoch:  5 | train loss: 567.6680\n",
      "Epoch:  6 | train loss: 56.8940\n",
      "Epoch:  7 | train loss: 47.8007\n",
      "Epoch:  8 | train loss: 67.4951\n",
      "Epoch:  9 | train loss: 66.5822\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 87.6715\n",
      "Epoch:  1 | train loss: 78.6350\n",
      "Epoch:  2 | train loss: 83.7297\n",
      "Epoch:  3 | train loss: 68.2845\n",
      "Epoch:  4 | train loss: 80.0141\n",
      "Epoch:  5 | train loss: 77.3084\n",
      "Epoch:  6 | train loss: 110.2526\n",
      "Epoch:  7 | train loss: 84.9434\n",
      "Epoch:  8 | train loss: 197.1666\n",
      "Epoch:  9 | train loss: 45.0897\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 136.3099\n",
      "Epoch:  1 | train loss: 51.9535\n",
      "Epoch:  2 | train loss: 113.1795\n",
      "Epoch:  3 | train loss: 165.7545\n",
      "Epoch:  4 | train loss: 77.9952\n",
      "Epoch:  5 | train loss: 130.7460\n",
      "Epoch:  6 | train loss: 106.3592\n",
      "Epoch:  7 | train loss: 89.5114\n",
      "Epoch:  8 | train loss: 108.8741\n",
      "Epoch:  9 | train loss: 232.4062\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 52.2337\n",
      "Epoch:  1 | train loss: 54.0463\n",
      "Epoch:  2 | train loss: 147.1490\n",
      "Epoch:  3 | train loss: 88.2215\n",
      "Epoch:  4 | train loss: 329.3222\n",
      "Epoch:  5 | train loss: 107.7926\n",
      "Epoch:  6 | train loss: 111.1777\n",
      "Epoch:  7 | train loss: 72.0156\n",
      "Epoch:  8 | train loss: 268.5566\n",
      "Epoch:  9 | train loss: 70.0943\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'QRNN', 'DPRNN']:\n",
    "    for seed in range(5):\n",
    "        run_medical_experiments(dataset='eeg', \n",
    "                                baseline=baseline,\n",
    "                                save_model=True, \n",
    "                                save_results=True,\n",
    "                                seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d8212e52-b753-4819-b28d-8e0a0f4f1dc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CFRNN\n",
      "96.5 \\(\\pm\\) 1.0\\%\n",
      "\n",
      "QRNN\n",
      "48.0 \\(\\pm\\) 4.0\\%\n",
      "\n",
      "DPRNN\n",
      "3.3 \\(\\pm\\) 0.7\\%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'QRNN', 'DPRNN']:\n",
    "    print(baseline)\n",
    "    coverages_mean, coverages_std = get_joint_medical_coverages(baseline, 'eeg', seeds=range(5))\n",
    "    \n",
    "    print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "39a4e009-d245-4c84-aa68-a2d5eb2e2a8b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CFRNN\n",
      "61.863339309692385\n",
      "18.023400935029958\n",
      "\n",
      "DPRNN\n",
      "7.387410955429077\n",
      "0.7367469770792188\n",
      "\n",
      "QRNN\n",
      "21.385921783447266\n",
      "2.356940865461466\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'DPRNN', 'QRNN']:\n",
    "    print(baseline)\n",
    "    widths_mean, widths_std = get_medical_interval_widths(baseline, 'eeg', seeds=range(5))\n",
    "    \n",
    "    print(widths_mean)\n",
    "    print(widths_std)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20370c7c-eaca-49aa-8e45-7f91c01c4faa",
   "metadata": {},
   "source": [
    "## COVID-19 dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "84cb28a9-05b8-4be6-9656-44cc61953beb",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 19849.4248046875\n",
      "Epoch: 50\tTrain loss: 14987.1103515625\n",
      "Epoch: 100\tTrain loss: 12784.7021484375\n",
      "Epoch: 150\tTrain loss: 16437.63427734375\n",
      "Epoch: 200\tTrain loss: 11722.68603515625\n",
      "Epoch: 250\tTrain loss: 15581.306396484375\n",
      "Epoch: 300\tTrain loss: 10658.34130859375\n",
      "Epoch: 350\tTrain loss: 8696.23583984375\n",
      "Epoch: 400\tTrain loss: 9109.50390625\n",
      "Epoch: 450\tTrain loss: 10849.78125\n",
      "Epoch: 500\tTrain loss: 11905.59814453125\n",
      "Epoch: 550\tTrain loss: 8048.343505859375\n",
      "Epoch: 600\tTrain loss: 7777.374267578125\n",
      "Epoch: 650\tTrain loss: 7038.0400390625\n",
      "Epoch: 700\tTrain loss: 10878.801025390625\n",
      "Epoch: 750\tTrain loss: 5562.97900390625\n",
      "Epoch: 800\tTrain loss: 5780.314453125\n",
      "Epoch: 850\tTrain loss: 6390.28076171875\n",
      "Epoch: 900\tTrain loss: 5894.050537109375\n",
      "Epoch: 950\tTrain loss: 10112.148315429688\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 22146.2509765625\n",
      "Epoch: 50\tTrain loss: 18750.36279296875\n",
      "Epoch: 100\tTrain loss: 16809.369140625\n",
      "Epoch: 150\tTrain loss: 14085.14453125\n",
      "Epoch: 200\tTrain loss: 13656.8427734375\n",
      "Epoch: 250\tTrain loss: 17567.69384765625\n",
      "Epoch: 300\tTrain loss: 12284.51220703125\n",
      "Epoch: 350\tTrain loss: 11821.515625\n",
      "Epoch: 400\tTrain loss: 11277.4990234375\n",
      "Epoch: 450\tTrain loss: 10640.63134765625\n",
      "Epoch: 500\tTrain loss: 10791.790283203125\n",
      "Epoch: 550\tTrain loss: 9560.468017578125\n",
      "Epoch: 600\tTrain loss: 9476.66259765625\n",
      "Epoch: 650\tTrain loss: 9824.21533203125\n",
      "Epoch: 700\tTrain loss: 8281.2197265625\n",
      "Epoch: 750\tTrain loss: 12478.849609375\n",
      "Epoch: 800\tTrain loss: 7586.025390625\n",
      "Epoch: 850\tTrain loss: 6525.350341796875\n",
      "Epoch: 900\tTrain loss: 7190.408447265625\n",
      "Epoch: 950\tTrain loss: 9779.423583984375\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 29443.4013671875\n",
      "Epoch: 50\tTrain loss: 17357.818359375\n",
      "Epoch: 100\tTrain loss: 18555.5595703125\n",
      "Epoch: 150\tTrain loss: 13638.90380859375\n",
      "Epoch: 200\tTrain loss: 13406.48876953125\n",
      "Epoch: 250\tTrain loss: 14239.46630859375\n",
      "Epoch: 300\tTrain loss: 12606.09521484375\n",
      "Epoch: 350\tTrain loss: 12864.31201171875\n",
      "Epoch: 400\tTrain loss: 11744.1630859375\n",
      "Epoch: 450\tTrain loss: 12568.56201171875\n",
      "Epoch: 500\tTrain loss: 10933.62646484375\n",
      "Epoch: 550\tTrain loss: 13946.4501953125\n",
      "Epoch: 600\tTrain loss: 8862.84033203125\n",
      "Epoch: 650\tTrain loss: 8769.62939453125\n",
      "Epoch: 700\tTrain loss: 11424.7958984375\n",
      "Epoch: 750\tTrain loss: 7796.840576171875\n",
      "Epoch: 800\tTrain loss: 11082.177490234375\n",
      "Epoch: 850\tTrain loss: 11320.429931640625\n",
      "Epoch: 900\tTrain loss: 6299.620849609375\n",
      "Epoch: 950\tTrain loss: 6850.492431640625\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 19979.0244140625\n",
      "Epoch: 50\tTrain loss: 16717.2666015625\n",
      "Epoch: 100\tTrain loss: 13970.81982421875\n",
      "Epoch: 150\tTrain loss: 13371.89453125\n",
      "Epoch: 200\tTrain loss: 12366.76025390625\n",
      "Epoch: 250\tTrain loss: 9907.839599609375\n",
      "Epoch: 300\tTrain loss: 10948.4892578125\n",
      "Epoch: 350\tTrain loss: 10148.15625\n",
      "Epoch: 400\tTrain loss: 11577.26611328125\n",
      "Epoch: 450\tTrain loss: 9296.483154296875\n",
      "Epoch: 500\tTrain loss: 10771.06201171875\n",
      "Epoch: 550\tTrain loss: 9124.300048828125\n",
      "Epoch: 600\tTrain loss: 7559.131103515625\n",
      "Epoch: 650\tTrain loss: 7010.19775390625\n",
      "Epoch: 700\tTrain loss: 7351.08251953125\n",
      "Epoch: 750\tTrain loss: 6783.565673828125\n",
      "Epoch: 800\tTrain loss: 6037.81689453125\n",
      "Epoch: 850\tTrain loss: 6214.803466796875\n",
      "Epoch: 900\tTrain loss: 5912.7353515625\n",
      "Epoch: 950\tTrain loss: 5232.92333984375\n",
      "Training CFRNN\n",
      "Epoch: 0\tTrain loss: 17277.6435546875\n",
      "Epoch: 50\tTrain loss: 14058.4990234375\n",
      "Epoch: 100\tTrain loss: 10642.69189453125\n",
      "Epoch: 150\tTrain loss: 9940.4501953125\n",
      "Epoch: 200\tTrain loss: 8698.67626953125\n",
      "Epoch: 250\tTrain loss: 8605.575927734375\n",
      "Epoch: 300\tTrain loss: 8713.2470703125\n",
      "Epoch: 350\tTrain loss: 6713.533447265625\n",
      "Epoch: 400\tTrain loss: 6298.8543701171875\n",
      "Epoch: 450\tTrain loss: 9026.096923828125\n",
      "Epoch: 500\tTrain loss: 6890.9462890625\n",
      "Epoch: 550\tTrain loss: 5680.34326171875\n",
      "Epoch: 600\tTrain loss: 5453.46533203125\n",
      "Epoch: 650\tTrain loss: 4968.895751953125\n",
      "Epoch: 700\tTrain loss: 4307.9295654296875\n",
      "Epoch: 750\tTrain loss: 3510.3655395507812\n",
      "Epoch: 800\tTrain loss: 4106.7841796875\n",
      "Epoch: 850\tTrain loss: 4016.16796875\n",
      "Epoch: 900\tTrain loss: 3501.784423828125\n",
      "Epoch: 950\tTrain loss: 4356.518310546875\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 48.7312\n",
      "Epoch:  1 | train loss: 27.3087\n",
      "Epoch:  2 | train loss: 18.9063\n",
      "Epoch:  3 | train loss: 13.2338\n",
      "Epoch:  4 | train loss: 12.1036\n",
      "Epoch:  5 | train loss: 8.2637\n",
      "Epoch:  6 | train loss: 15.8762\n",
      "Epoch:  7 | train loss: 8.3898\n",
      "Epoch:  8 | train loss: 8.0952\n",
      "Epoch:  9 | train loss: 11.6282\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 28.6959\n",
      "Epoch:  1 | train loss: 24.3513\n",
      "Epoch:  2 | train loss: 20.9485\n",
      "Epoch:  3 | train loss: 13.0802\n",
      "Epoch:  4 | train loss: 9.0241\n",
      "Epoch:  5 | train loss: 9.2148\n",
      "Epoch:  6 | train loss: 8.7758\n",
      "Epoch:  7 | train loss: 11.6092\n",
      "Epoch:  8 | train loss: 11.7043\n",
      "Epoch:  9 | train loss: 9.9961\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 30.6576\n",
      "Epoch:  1 | train loss: 26.4580\n",
      "Epoch:  2 | train loss: 19.1201\n",
      "Epoch:  3 | train loss: 16.5156\n",
      "Epoch:  4 | train loss: 12.6003\n",
      "Epoch:  5 | train loss: 11.2447\n",
      "Epoch:  6 | train loss: 10.8522\n",
      "Epoch:  7 | train loss: 8.1644\n",
      "Epoch:  8 | train loss: 11.0160\n",
      "Epoch:  9 | train loss: 10.4930\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 27.0085\n",
      "Epoch:  1 | train loss: 19.5236\n",
      "Epoch:  2 | train loss: 19.3575\n",
      "Epoch:  3 | train loss: 13.1351\n",
      "Epoch:  4 | train loss: 11.6412\n",
      "Epoch:  5 | train loss: 9.4715\n",
      "Epoch:  6 | train loss: 10.1316\n",
      "Epoch:  7 | train loss: 8.3316\n",
      "Epoch:  8 | train loss: 9.1588\n",
      "Epoch:  9 | train loss: 8.9913\n",
      "Training QRNN\n",
      "Epoch:  0 | train loss: 26.4044\n",
      "Epoch:  1 | train loss: 24.4085\n",
      "Epoch:  2 | train loss: 21.2035\n",
      "Epoch:  3 | train loss: 11.8318\n",
      "Epoch:  4 | train loss: 12.2787\n",
      "Epoch:  5 | train loss: 12.5255\n",
      "Epoch:  6 | train loss: 12.7370\n",
      "Epoch:  7 | train loss: 24.2223\n",
      "Epoch:  8 | train loss: 12.8108\n",
      "Epoch:  9 | train loss: 13.8540\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 12160.6113\n",
      "Epoch:  1 | train loss: 3605.4265\n",
      "Epoch:  2 | train loss: 4614.0601\n",
      "Epoch:  3 | train loss: 3544.8232\n",
      "Epoch:  4 | train loss: 3023.8994\n",
      "Epoch:  5 | train loss: 5363.6377\n",
      "Epoch:  6 | train loss: 3092.2188\n",
      "Epoch:  7 | train loss: 5378.3447\n",
      "Epoch:  8 | train loss: 7205.3936\n",
      "Epoch:  9 | train loss: 4649.1826\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 8378.7734\n",
      "Epoch:  1 | train loss: 5759.3452\n",
      "Epoch:  2 | train loss: 3610.5566\n",
      "Epoch:  3 | train loss: 3209.2981\n",
      "Epoch:  4 | train loss: 4308.8535\n",
      "Epoch:  5 | train loss: 4325.1064\n",
      "Epoch:  6 | train loss: 5464.8872\n",
      "Epoch:  7 | train loss: 3907.5706\n",
      "Epoch:  8 | train loss: 3839.8362\n",
      "Epoch:  9 | train loss: 6803.8638\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 7760.9771\n",
      "Epoch:  1 | train loss: 5146.1055\n",
      "Epoch:  2 | train loss: 7603.5601\n",
      "Epoch:  3 | train loss: 6824.4819\n",
      "Epoch:  4 | train loss: 3987.3645\n",
      "Epoch:  5 | train loss: 2439.1169\n",
      "Epoch:  6 | train loss: 5312.8813\n",
      "Epoch:  7 | train loss: 3780.3679\n",
      "Epoch:  8 | train loss: 5400.5771\n",
      "Epoch:  9 | train loss: 3755.9587\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 9112.7441\n",
      "Epoch:  1 | train loss: 4554.3262\n",
      "Epoch:  2 | train loss: 3647.7119\n",
      "Epoch:  3 | train loss: 4283.5596\n",
      "Epoch:  4 | train loss: 3775.8174\n",
      "Epoch:  5 | train loss: 3132.4783\n",
      "Epoch:  6 | train loss: 3233.6782\n",
      "Epoch:  7 | train loss: 3145.4329\n",
      "Epoch:  8 | train loss: 3023.9573\n",
      "Epoch:  9 | train loss: 2512.9871\n",
      "Training DPRNN\n",
      "Epoch:  0 | train loss: 8818.3545\n",
      "Epoch:  1 | train loss: 4829.1543\n",
      "Epoch:  2 | train loss: 3680.0000\n",
      "Epoch:  3 | train loss: 10231.6299\n",
      "Epoch:  4 | train loss: 2623.3699\n",
      "Epoch:  5 | train loss: 6541.4731\n",
      "Epoch:  6 | train loss: 6590.5640\n",
      "Epoch:  7 | train loss: 2725.9817\n",
      "Epoch:  8 | train loss: 4059.1567\n",
      "Epoch:  9 | train loss: 2644.1416\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'QRNN', 'DPRNN']:\n",
    "    for seed in range(5):\n",
    "        run_medical_experiments(dataset='covid', \n",
    "                                baseline=baseline,\n",
    "                                save_model=True, \n",
    "                                save_results=True,\n",
    "                                seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a45a98a5-3175-40ce-9c70-5c0b236300b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CFRNN\n",
      "89.7 \\(\\pm\\) 5.3\\%\n",
      "\n",
      "QRNN\n",
      "15.0 \\(\\pm\\) 5.9\\%\n",
      "\n",
      "DPRNN\n",
      "0.0 \\(\\pm\\) 0.0\\%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'QRNN', 'DPRNN']:\n",
    "    print(baseline)\n",
    "    coverages_mean, coverages_std = get_joint_medical_coverages(baseline, 'covid', seeds=range(5))\n",
    "    \n",
    "    print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4074c07c-9cca-49c8-83b3-156d108530a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CFRNN\n",
      "733.9547253723144\n",
      "582.5152458938113\n",
      "\n",
      "DPRNN\n",
      "61.18421086502075\n",
      "32.372609877768895\n",
      "\n",
      "QRNN\n",
      "136.56350823974608\n",
      "63.3235278998503\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for baseline in ['CFRNN', 'DPRNN', 'QRNN']:\n",
    "    print(baseline)\n",
    "    widths_mean, widths_std = get_medical_interval_widths(baseline, 'covid', seeds=range(5))\n",
    "    \n",
    "    print(widths_mean)\n",
    "    print(widths_std)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0b6d7af-a7c6-46eb-aead-23336fa9a665",
   "metadata": {},
   "source": [
    "## Ablation: Uncorrected calibration scores"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4eae9c5f-0688-4e0b-83da-b00fc3ae4cde",
   "metadata": {},
   "source": [
    "#### MIMIC-III"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "01820b4f-3959-46b2-8632-81f6df4bfac7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94.0 \\(\\pm\\) 1.2\\%\n"
     ]
    }
   ],
   "source": [
    "coverages_mean, coverages_std = get_joint_medical_coverages('CFRNN', 'mimic', seeds=range(5), correct_conformal=True)\n",
    "    \n",
    "print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "121b7032-4fb5-4bd1-a629-4fe78801cdd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89.0 \\(\\pm\\) 1.4\\%\n"
     ]
    }
   ],
   "source": [
    "coverages_mean, coverages_std = get_joint_medical_coverages('CFRNN', 'mimic', seeds=range(5), correct_conformal=False)\n",
    "    \n",
    "print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "16f876f3-40be-46df-8fd0-fb08261ebfdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[94.0\\%, 94.0\\%]\n",
      "[95.2\\%, 95.4\\%]\n",
      "[94.0\\%, 95.2\\%]\n",
      "[96.0\\%, 96.8\\%]\n",
      "[93.8\\%, 94.0\\%]\n"
     ]
    }
   ],
   "source": [
    "for seed in range(5):\n",
    "    results = load_medical_results(dataset='mimic', baseline='CFRNN', seed=seed)\n",
    "    independent_coverages = results['Mean independent coverage']\n",
    "    print('[{:.1f}\\\\%, {:.1f}\\\\%]'.format(independent_coverages.min() * 100, independent_coverages.max() * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "ff4445d1-e5f3-46d2-a7b4-edf517c67d95",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[89.4\\%, 89.8\\%]\n",
      "[91.2\\%, 91.4\\%]\n",
      "[89.0\\%, 90.0\\%]\n",
      "[90.8\\%, 91.4\\%]\n",
      "[90.4\\%, 91.2\\%]\n"
     ]
    }
   ],
   "source": [
    "for seed in range(5):\n",
    "    uncorrected_mimic_results = get_uncorrected_medical_results(dataset='mimic', seed=seed)\n",
    "    independent_coverages = uncorrected_mimic_results['Mean independent coverage']\n",
    "    print('[{:.1f}\\\\%, {:.1f}\\\\%]'.format(independent_coverages.min() * 100, independent_coverages.max() * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55ab9a25-464b-41d1-9a66-3dc0389c301e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d146c828-ecf1-447b-a3ba-aa511f6b28b6",
   "metadata": {},
   "source": [
    "#### EEG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "02e071be-2b6c-4fae-83cd-97d32c6f0f2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96.5 \\(\\pm\\) 1.0\\%\n"
     ]
    }
   ],
   "source": [
    "coverages_mean, coverages_std = get_joint_medical_coverages('CFRNN', 'eeg', seeds=range(5), correct_conformal=True)\n",
    "    \n",
    "print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "43c3db39-0219-4eef-9f45-22d751ba8861",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59.4 \\(\\pm\\) 2.4\\%\n"
     ]
    }
   ],
   "source": [
    "coverages_mean, coverages_std = get_joint_medical_coverages('CFRNN', 'eeg', seeds=range(5), correct_conformal=False)\n",
    "    \n",
    "print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c53ee6a6-12e3-4ab6-aa78-50dfee1d9d29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[98.5\\%, 99.8\\%]\n",
      "[98.5\\%, 99.5\\%]\n",
      "[98.3\\%, 99.7\\%]\n",
      "[98.9\\%, 99.8\\%]\n",
      "[98.3\\%, 99.3\\%]\n"
     ]
    }
   ],
   "source": [
    "for seed in range(5):\n",
    "    results = load_medical_results(dataset='eeg', baseline='CFRNN', seed=seed)\n",
    "    independent_coverages = results['Mean independent coverage']\n",
    "    print('[{:.1f}\\\\%, {:.1f}\\\\%]'.format(independent_coverages.min() * 100, independent_coverages.max() * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "45bcfc90-6918-4461-ab24-adeccaee56a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[86.3\\%, 91.4\\%]\n",
      "[85.7\\%, 91.2\\%]\n",
      "[86.0\\%, 90.6\\%]\n",
      "[87.4\\%, 91.6\\%]\n",
      "[85.5\\%, 90.8\\%]\n"
     ]
    }
   ],
   "source": [
    "for seed in range(5):\n",
    "    uncorrected_mimic_results = get_uncorrected_medical_results(dataset='eeg', seed=seed)\n",
    "    independent_coverages = uncorrected_mimic_results['Mean independent coverage']\n",
    "    print('[{:.1f}\\\\%, {:.1f}\\\\%]'.format(independent_coverages.min() * 100, independent_coverages.max() * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c58171e-7675-4983-851c-5a963d25aeec",
   "metadata": {},
   "source": [
    "#### COVID-19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "752fa9ad-f6cc-43d3-9fb5-5bfdb071888e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "89.7 \\(\\pm\\) 5.3\\%\n"
     ]
    }
   ],
   "source": [
    "coverages_mean, coverages_std = get_joint_medical_coverages('CFRNN', 'covid', seeds=range(5), correct_conformal=True)\n",
    "    \n",
    "print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "86d905e4-dc31-47ae-9b47-bda58c4c59aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.5 \\(\\pm\\) 8.0\\%\n"
     ]
    }
   ],
   "source": [
    "coverages_mean, coverages_std = get_joint_medical_coverages('CFRNN', 'covid', seeds=range(5), correct_conformal=False)\n",
    "    \n",
    "print('{:.1f} \\\\(\\\\pm\\\\) {:.1f}\\\\%'.format(coverages_mean, coverages_std))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "66c5f5a2-4423-4aff-b2e4-91cc7a3ce3be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[95.0\\%, 100.0\\%]\n",
      "[93.8\\%, 100.0\\%]\n",
      "[96.2\\%, 100.0\\%]\n",
      "[87.5\\%, 98.8\\%]\n",
      "[96.2\\%, 100.0\\%]\n"
     ]
    }
   ],
   "source": [
    "for seed in range(5):\n",
    "    results = load_medical_results(dataset='covid', baseline='CFRNN', seed=seed)\n",
    "    independent_coverages = results['Mean independent coverage']\n",
    "    print('[{:.1f}\\\\%, {:.1f}\\\\%]'.format(independent_coverages.min() * 100, independent_coverages.max() * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ab9b269d-13d6-4f21-8848-d6eaeb6bc8cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[81.2\\%, 98.8\\%]\n",
      "[81.2\\%, 98.8\\%]\n",
      "[87.5\\%, 98.8\\%]\n",
      "[77.5\\%, 95.0\\%]\n",
      "[85.0\\%, 96.2\\%]\n"
     ]
    }
   ],
   "source": [
    "for seed in range(5):\n",
    "    uncorrected_mimic_results = get_uncorrected_medical_results(dataset='covid', seed=seed)\n",
    "    independent_coverages = uncorrected_mimic_results['Mean independent coverage']\n",
    "    print('[{:.1f}\\\\%, {:.1f}\\\\%]'.format(independent_coverages.min() * 100, independent_coverages.max() * 100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}