Example_Application.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DkrdV_YzwWPR"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6965,
     "status": "ok",
     "timestamp": 1699865202101,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "C1tpx4Hug427",
    "outputId": "fcebe1f4-b0e3-4401-c720-25b205812188"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "import sys\n",
    "sys.path.insert(0,'/content/drive/MyDrive/Confounder_Removal_Functions')\n",
    "import Confounder_Correction_Classes\n",
    "from Confounder_Correction_Classes import ComBatHarmonization,StandardScalerDict, BiocovariatesRegression\n",
    "\n",
    "#Datasets and system\n",
    "import numpy as np                # Management of arrays\n",
    "import os                         # System utils\n",
    "import pandas as pd\n",
    "from pathlib import Path          # path and file utils\n",
    "from scipy.io import loadmat      # Required to load .mat files\n",
    "import h5py #For creating new dataset files\n",
    "import sklearn\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "o6aDyof8xBdQ"
   },
   "outputs": [],
   "source": [
    "# Atlas regions names\n",
    "data_path='/content/drive/MyDrive/Confounder_Removal_Functions/'\n",
    "\n",
    "# StratiBip\n",
    "Strat_covars_raw=pd.read_csv(os.path.join(data_path, \"StratiBip_YA_covar_raw.csv\"))\n",
    "Strat_data_raw=pd.read_csv(os.path.join(data_path, \"StratiBip_YA_data_raw.csv\"))\n",
    "\n",
    "n_Strat=Strat_data_raw.shape[0]\n",
    "\n",
    "# Globals\n",
    "\n",
    "strat_globals_data=copy.deepcopy(Strat_covars_raw[['TIV','CT_avg','CT_std','abGM','abWM','abCSF']])\n",
    "strat_covars=copy.deepcopy(Strat_covars_raw[['Batch','Age','Gender','Diagnosis','index']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 443
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1699866316056,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "CepSeIz-7SwD",
    "outputId": "6dfca7a7-411e-46dc-d138-6f335842d759"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-b2c88055-3291-48e2-96d1-a9a0e04ed0c7\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>308</th>\n",
       "      <th>309</th>\n",
       "      <th>310</th>\n",
       "      <th>311</th>\n",
       "      <th>312</th>\n",
       "      <th>313</th>\n",
       "      <th>314</th>\n",
       "      <th>315</th>\n",
       "      <th>316</th>\n",
       "      <th>317</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.381284</td>\n",
       "      <td>2.365205</td>\n",
       "      <td>2.394465</td>\n",
       "      <td>2.390131</td>\n",
       "      <td>2.296173</td>\n",
       "      <td>2.363262</td>\n",
       "      <td>1.842687</td>\n",
       "      <td>1.945240</td>\n",
       "      <td>3.641180</td>\n",
       "      <td>3.911366</td>\n",
       "      <td>...</td>\n",
       "      <td>2.256169</td>\n",
       "      <td>2.483417</td>\n",
       "      <td>2.149909</td>\n",
       "      <td>2.283767</td>\n",
       "      <td>2.259032</td>\n",
       "      <td>2.390745</td>\n",
       "      <td>2.320724</td>\n",
       "      <td>2.386808</td>\n",
       "      <td>2.522325</td>\n",
       "      <td>2.064180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.535161</td>\n",
       "      <td>2.597305</td>\n",
       "      <td>2.370064</td>\n",
       "      <td>2.224272</td>\n",
       "      <td>2.712814</td>\n",
       "      <td>2.635750</td>\n",
       "      <td>1.990101</td>\n",
       "      <td>2.122013</td>\n",
       "      <td>2.229588</td>\n",
       "      <td>3.995840</td>\n",
       "      <td>...</td>\n",
       "      <td>2.316820</td>\n",
       "      <td>2.733729</td>\n",
       "      <td>2.395556</td>\n",
       "      <td>2.577311</td>\n",
       "      <td>2.432763</td>\n",
       "      <td>2.456375</td>\n",
       "      <td>2.554963</td>\n",
       "      <td>2.586361</td>\n",
       "      <td>2.428077</td>\n",
       "      <td>2.293338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.476916</td>\n",
       "      <td>2.783529</td>\n",
       "      <td>2.163917</td>\n",
       "      <td>2.158305</td>\n",
       "      <td>2.393425</td>\n",
       "      <td>2.396675</td>\n",
       "      <td>1.910074</td>\n",
       "      <td>2.004064</td>\n",
       "      <td>3.483266</td>\n",
       "      <td>3.633593</td>\n",
       "      <td>...</td>\n",
       "      <td>2.224061</td>\n",
       "      <td>2.227433</td>\n",
       "      <td>2.286425</td>\n",
       "      <td>2.403570</td>\n",
       "      <td>2.291334</td>\n",
       "      <td>2.237200</td>\n",
       "      <td>2.526054</td>\n",
       "      <td>2.565900</td>\n",
       "      <td>2.164334</td>\n",
       "      <td>2.199235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.747808</td>\n",
       "      <td>2.984173</td>\n",
       "      <td>2.282731</td>\n",
       "      <td>2.393531</td>\n",
       "      <td>2.724758</td>\n",
       "      <td>2.809444</td>\n",
       "      <td>1.877798</td>\n",
       "      <td>1.938598</td>\n",
       "      <td>3.557219</td>\n",
       "      <td>3.614868</td>\n",
       "      <td>...</td>\n",
       "      <td>2.547008</td>\n",
       "      <td>1.794572</td>\n",
       "      <td>2.467627</td>\n",
       "      <td>2.472297</td>\n",
       "      <td>2.597437</td>\n",
       "      <td>2.456284</td>\n",
       "      <td>2.698962</td>\n",
       "      <td>2.855816</td>\n",
       "      <td>2.211158</td>\n",
       "      <td>2.685730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.592410</td>\n",
       "      <td>2.839498</td>\n",
       "      <td>2.528011</td>\n",
       "      <td>2.633329</td>\n",
       "      <td>2.663850</td>\n",
       "      <td>2.601842</td>\n",
       "      <td>2.088841</td>\n",
       "      <td>2.124148</td>\n",
       "      <td>3.607335</td>\n",
       "      <td>3.360040</td>\n",
       "      <td>...</td>\n",
       "      <td>2.360878</td>\n",
       "      <td>2.341806</td>\n",
       "      <td>2.358478</td>\n",
       "      <td>2.458825</td>\n",
       "      <td>2.372120</td>\n",
       "      <td>2.505500</td>\n",
       "      <td>2.655594</td>\n",
       "      <td>2.687394</td>\n",
       "      <td>2.094476</td>\n",
       "      <td>2.074966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>2.610727</td>\n",
       "      <td>2.465406</td>\n",
       "      <td>2.464656</td>\n",
       "      <td>2.277703</td>\n",
       "      <td>2.561621</td>\n",
       "      <td>2.647865</td>\n",
       "      <td>1.963910</td>\n",
       "      <td>2.038170</td>\n",
       "      <td>3.357151</td>\n",
       "      <td>3.962732</td>\n",
       "      <td>...</td>\n",
       "      <td>2.801013</td>\n",
       "      <td>2.553129</td>\n",
       "      <td>2.628645</td>\n",
       "      <td>2.626510</td>\n",
       "      <td>2.322253</td>\n",
       "      <td>2.512811</td>\n",
       "      <td>2.610389</td>\n",
       "      <td>2.636797</td>\n",
       "      <td>2.203861</td>\n",
       "      <td>2.336355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>2.457676</td>\n",
       "      <td>2.359974</td>\n",
       "      <td>2.282763</td>\n",
       "      <td>2.441842</td>\n",
       "      <td>2.413875</td>\n",
       "      <td>2.351588</td>\n",
       "      <td>1.921241</td>\n",
       "      <td>1.922233</td>\n",
       "      <td>3.360151</td>\n",
       "      <td>3.672949</td>\n",
       "      <td>...</td>\n",
       "      <td>2.247912</td>\n",
       "      <td>2.359588</td>\n",
       "      <td>2.130224</td>\n",
       "      <td>2.290067</td>\n",
       "      <td>2.472081</td>\n",
       "      <td>2.431388</td>\n",
       "      <td>2.473254</td>\n",
       "      <td>2.373903</td>\n",
       "      <td>1.941634</td>\n",
       "      <td>2.365668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>2.534062</td>\n",
       "      <td>2.630353</td>\n",
       "      <td>2.570794</td>\n",
       "      <td>2.278625</td>\n",
       "      <td>2.612592</td>\n",
       "      <td>2.578483</td>\n",
       "      <td>2.042885</td>\n",
       "      <td>2.034513</td>\n",
       "      <td>3.893594</td>\n",
       "      <td>3.702153</td>\n",
       "      <td>...</td>\n",
       "      <td>2.208838</td>\n",
       "      <td>2.545933</td>\n",
       "      <td>2.398403</td>\n",
       "      <td>2.327816</td>\n",
       "      <td>2.347026</td>\n",
       "      <td>2.510887</td>\n",
       "      <td>2.431066</td>\n",
       "      <td>2.566320</td>\n",
       "      <td>2.104877</td>\n",
       "      <td>2.203844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>2.316958</td>\n",
       "      <td>2.556639</td>\n",
       "      <td>2.555630</td>\n",
       "      <td>2.262312</td>\n",
       "      <td>2.622276</td>\n",
       "      <td>2.730452</td>\n",
       "      <td>2.001902</td>\n",
       "      <td>1.937405</td>\n",
       "      <td>4.022779</td>\n",
       "      <td>3.651444</td>\n",
       "      <td>...</td>\n",
       "      <td>2.202630</td>\n",
       "      <td>2.094916</td>\n",
       "      <td>2.478279</td>\n",
       "      <td>2.420420</td>\n",
       "      <td>2.377548</td>\n",
       "      <td>2.454765</td>\n",
       "      <td>2.400840</td>\n",
       "      <td>2.596233</td>\n",
       "      <td>2.382636</td>\n",
       "      <td>2.503186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>2.449882</td>\n",
       "      <td>2.658629</td>\n",
       "      <td>2.400186</td>\n",
       "      <td>2.350265</td>\n",
       "      <td>2.617003</td>\n",
       "      <td>2.533531</td>\n",
       "      <td>2.051997</td>\n",
       "      <td>2.080512</td>\n",
       "      <td>4.476471</td>\n",
       "      <td>4.510477</td>\n",
       "      <td>...</td>\n",
       "      <td>2.602858</td>\n",
       "      <td>2.514718</td>\n",
       "      <td>2.358062</td>\n",
       "      <td>2.444621</td>\n",
       "      <td>2.433539</td>\n",
       "      <td>2.544375</td>\n",
       "      <td>2.529172</td>\n",
       "      <td>2.592109</td>\n",
       "      <td>2.435130</td>\n",
       "      <td>2.292646</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>563 rows × 318 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b2c88055-3291-48e2-96d1-a9a0e04ed0c7')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-b2c88055-3291-48e2-96d1-a9a0e04ed0c7 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-b2c88055-3291-48e2-96d1-a9a0e04ed0c7');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-0f156899-d563-44e5-9712-632e140061d6\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0f156899-d563-44e5-9712-632e140061d6')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-0f156899-d563-44e5-9712-632e140061d6 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "            0         1         2         3         4         5         6  \\\n",
       "0    2.381284  2.365205  2.394465  2.390131  2.296173  2.363262  1.842687   \n",
       "1    2.535161  2.597305  2.370064  2.224272  2.712814  2.635750  1.990101   \n",
       "2    2.476916  2.783529  2.163917  2.158305  2.393425  2.396675  1.910074   \n",
       "3    2.747808  2.984173  2.282731  2.393531  2.724758  2.809444  1.877798   \n",
       "4    2.592410  2.839498  2.528011  2.633329  2.663850  2.601842  2.088841   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "558  2.610727  2.465406  2.464656  2.277703  2.561621  2.647865  1.963910   \n",
       "559  2.457676  2.359974  2.282763  2.441842  2.413875  2.351588  1.921241   \n",
       "560  2.534062  2.630353  2.570794  2.278625  2.612592  2.578483  2.042885   \n",
       "561  2.316958  2.556639  2.555630  2.262312  2.622276  2.730452  2.001902   \n",
       "562  2.449882  2.658629  2.400186  2.350265  2.617003  2.533531  2.051997   \n",
       "\n",
       "            7         8         9  ...       308       309       310  \\\n",
       "0    1.945240  3.641180  3.911366  ...  2.256169  2.483417  2.149909   \n",
       "1    2.122013  2.229588  3.995840  ...  2.316820  2.733729  2.395556   \n",
       "2    2.004064  3.483266  3.633593  ...  2.224061  2.227433  2.286425   \n",
       "3    1.938598  3.557219  3.614868  ...  2.547008  1.794572  2.467627   \n",
       "4    2.124148  3.607335  3.360040  ...  2.360878  2.341806  2.358478   \n",
       "..        ...       ...       ...  ...       ...       ...       ...   \n",
       "558  2.038170  3.357151  3.962732  ...  2.801013  2.553129  2.628645   \n",
       "559  1.922233  3.360151  3.672949  ...  2.247912  2.359588  2.130224   \n",
       "560  2.034513  3.893594  3.702153  ...  2.208838  2.545933  2.398403   \n",
       "561  1.937405  4.022779  3.651444  ...  2.202630  2.094916  2.478279   \n",
       "562  2.080512  4.476471  4.510477  ...  2.602858  2.514718  2.358062   \n",
       "\n",
       "          311       312       313       314       315       316       317  \n",
       "0    2.283767  2.259032  2.390745  2.320724  2.386808  2.522325  2.064180  \n",
       "1    2.577311  2.432763  2.456375  2.554963  2.586361  2.428077  2.293338  \n",
       "2    2.403570  2.291334  2.237200  2.526054  2.565900  2.164334  2.199235  \n",
       "3    2.472297  2.597437  2.456284  2.698962  2.855816  2.211158  2.685730  \n",
       "4    2.458825  2.372120  2.505500  2.655594  2.687394  2.094476  2.074966  \n",
       "..        ...       ...       ...       ...       ...       ...       ...  \n",
       "558  2.626510  2.322253  2.512811  2.610389  2.636797  2.203861  2.336355  \n",
       "559  2.290067  2.472081  2.431388  2.473254  2.373903  1.941634  2.365668  \n",
       "560  2.327816  2.347026  2.510887  2.431066  2.566320  2.104877  2.203844  \n",
       "561  2.420420  2.377548  2.454765  2.400840  2.596233  2.382636  2.503186  \n",
       "562  2.444621  2.433539  2.544375  2.529172  2.592109  2.435130  2.292646  \n",
       "\n",
       "[563 rows x 318 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Strat_data_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1699866288671,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "CU3Xlrxc7Q-3",
    "outputId": "8744ef76-9fee-4543-dbc7-daf56a960a4e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-7cabf5d9-166f-414e-959b-a8675748b8eb\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIV</th>\n",
       "      <th>CT_avg</th>\n",
       "      <th>CT_std</th>\n",
       "      <th>abGM</th>\n",
       "      <th>abWM</th>\n",
       "      <th>abCSF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1574.427875</td>\n",
       "      <td>2.309652</td>\n",
       "      <td>0.631975</td>\n",
       "      <td>719.357596</td>\n",
       "      <td>621.762847</td>\n",
       "      <td>233.307431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1349.838482</td>\n",
       "      <td>2.422376</td>\n",
       "      <td>0.652034</td>\n",
       "      <td>663.725497</td>\n",
       "      <td>477.630045</td>\n",
       "      <td>208.482940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1270.768332</td>\n",
       "      <td>2.284354</td>\n",
       "      <td>0.616954</td>\n",
       "      <td>619.703058</td>\n",
       "      <td>484.349787</td>\n",
       "      <td>166.715487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1451.979467</td>\n",
       "      <td>2.500177</td>\n",
       "      <td>0.669420</td>\n",
       "      <td>690.557075</td>\n",
       "      <td>537.012925</td>\n",
       "      <td>224.409467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1349.789110</td>\n",
       "      <td>2.478853</td>\n",
       "      <td>0.635754</td>\n",
       "      <td>638.354929</td>\n",
       "      <td>506.299478</td>\n",
       "      <td>205.134702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>1460.725141</td>\n",
       "      <td>2.546201</td>\n",
       "      <td>0.646318</td>\n",
       "      <td>718.589357</td>\n",
       "      <td>504.514549</td>\n",
       "      <td>237.621235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>1401.777282</td>\n",
       "      <td>2.322351</td>\n",
       "      <td>0.579651</td>\n",
       "      <td>629.240145</td>\n",
       "      <td>507.918718</td>\n",
       "      <td>264.618420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>1445.741133</td>\n",
       "      <td>2.457144</td>\n",
       "      <td>0.625968</td>\n",
       "      <td>639.810569</td>\n",
       "      <td>477.925325</td>\n",
       "      <td>328.005239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>1356.754475</td>\n",
       "      <td>2.464470</td>\n",
       "      <td>0.636989</td>\n",
       "      <td>653.460098</td>\n",
       "      <td>474.740553</td>\n",
       "      <td>228.553824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>1524.997435</td>\n",
       "      <td>2.512871</td>\n",
       "      <td>0.643172</td>\n",
       "      <td>747.923957</td>\n",
       "      <td>543.867906</td>\n",
       "      <td>233.205573</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>563 rows × 6 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7cabf5d9-166f-414e-959b-a8675748b8eb')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-7cabf5d9-166f-414e-959b-a8675748b8eb button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-7cabf5d9-166f-414e-959b-a8675748b8eb');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-c59bfe1f-7e64-4c61-86df-e26722c781f9\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c59bfe1f-7e64-4c61-86df-e26722c781f9')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-c59bfe1f-7e64-4c61-86df-e26722c781f9 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "             TIV    CT_avg    CT_std        abGM        abWM       abCSF\n",
       "0    1574.427875  2.309652  0.631975  719.357596  621.762847  233.307431\n",
       "1    1349.838482  2.422376  0.652034  663.725497  477.630045  208.482940\n",
       "2    1270.768332  2.284354  0.616954  619.703058  484.349787  166.715487\n",
       "3    1451.979467  2.500177  0.669420  690.557075  537.012925  224.409467\n",
       "4    1349.789110  2.478853  0.635754  638.354929  506.299478  205.134702\n",
       "..           ...       ...       ...         ...         ...         ...\n",
       "558  1460.725141  2.546201  0.646318  718.589357  504.514549  237.621235\n",
       "559  1401.777282  2.322351  0.579651  629.240145  507.918718  264.618420\n",
       "560  1445.741133  2.457144  0.625968  639.810569  477.925325  328.005239\n",
       "561  1356.754475  2.464470  0.636989  653.460098  474.740553  228.553824\n",
       "562  1524.997435  2.512871  0.643172  747.923957  543.867906  233.205573\n",
       "\n",
       "[563 rows x 6 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strat_globals_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1699866281409,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "18-6qDno7PNG",
    "outputId": "7657c99a-3924-4786-8243-c23dd0fba116"
   },
   "outputs": [],
   "source": [
    "strat_covars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6R4ate4A06aG"
   },
   "source": [
    "# Explaning ComBatHarmonization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Mgkaf5LPxPqK"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "------ComBatHarmonization Attributes--------------------------------------------\n",
    "\n",
    " cv_method=None : this options is not finished\n",
    "\n",
    " ref_batch= ref_batch: if ref_batch exist, is an integer corresponding to batch id, otherwise ref_batch=None\n",
    "\n",
    " regression_fit: whether the data will be used in the BiocovRegression afterwards.\n",
    "                 0 -> outputs the harmonized data only array (n_subj x n_feat);\n",
    "                 1 -> outputs a dictionary of the form of the input with harmonize\n",
    "                 data and respective covariates {'data': harmonize_output ; 'covariates': covars_matrix}\n",
    "                 ready for the linear regression fit.\n",
    "\n",
    " feat_detail: The details of the features to harmonize which are organized in a dictionary of dictionaries\n",
    "              Each first level of the dictionary corresponds to a set of features of the same name, e.g: Cortical thickness\n",
    "              , Volumes, surface areas, curvature; for each of this first level a second within dictionary is defined with\n",
    "              the column id of those features in the dataset 'id' and the corresponding biocovariates to preserve\n",
    "              in case there is none, define 'categorical' or 'continuous' with a empty list (eg: 'categorical': [] )\n",
    "\n",
    "              dict= {'name_of_feature_type_1: {'id': columns_id_corresponding,\n",
    "              'categorical': list_of_names_of_column_in_dataframe_of_categorical_covaariates,\n",
    "              'continuous': list_of_names_of_column_in_dataframe_of_continuous_covaariates }\n",
    "\n",
    "              -each of the levels of feature_type_i is harmonized independently of the others;\n",
    "\n",
    "feat_of_no_interest: better not to use this option for now, but it was to give\n",
    "                     the covariates to harmonize before harmonizing the main features.\n",
    "-------Input--------------------------------------------------------------------\n",
    "\n",
    "dictionary_input: a dictionary with 2 keys, 'data' and 'covariates'.\n",
    "                  'data': a dataframe (n_subjxn_feat)\n",
    "                  'covariates' a dataframe (n_subj x n_covariates) -> the columns\n",
    "                  in here need to have the names of biocovariates consistent with\n",
    "                  the ones in feat_detail 'continuous' and 'categorical'. It needs\n",
    "                  also the batch columns with the batch id's as integers! The batch\n",
    "                  columns needs to be named exactly 'batch' (see above strat_covars\n",
    "                  dataframe for a concrete example).\n",
    "------Ouput---------------------------------------------------------------------\n",
    "Depending on the regression_fit option:\n",
    "\n",
    "regression_fit: 0 -> outputs the harmonized data only array (n_subj x n_feat);\n",
    "                 1 -> outputs a dictionary of the form of the input with harmonize\n",
    "                 data and respective covariates {'data': harmonize_output ; 'covariates': covars_matrix}\n",
    "\n",
    "\"\"\"\n",
    "# feat_detail is the information related to the features to harmonized:\n",
    "# corresponding columns in dataset and respective biocovariates to consider\n",
    "\n",
    "cortical_columns= np.arange(0,68) # 68 cortical thickness DK40\n",
    "gvolumes_columns=np.arange(68,118) # 50 gray matter volumes CoBraLab\n",
    "wvolumes_columns=np.arange(118,170) # 52 white mater volumes CoBraLab\n",
    "volumes_columns=np.concatenate((gvolumes_columns,wvolumes_columns)) # all volumes\n",
    "\n",
    "feat_detail={'cortical_thickness': {'id': cortical_columns,\n",
    "                                                   'categorical': ['Gender'],\n",
    "                                                   'continuous':['Age']},\n",
    "              'volumes':           {'id': volumes_columns,\n",
    "                                        'categorical': ['Gender'],\n",
    "                                        'continuous':['Age', 'harmonized_TIV']}}\n",
    "\n",
    "define_combat_function=ComBatHarmonization(cv_method=None, ref_batch=None,\n",
    "                                           regression_fit=0,\n",
    "                                           feat_detail=feat_detail,\n",
    "                                           feat_of_no_interest=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Z4a6nrOf3KKd"
   },
   "source": [
    "# Explaining BiocovRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "h_7lScDo3Nrg"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "------BiocovRegression Attributes--------------------------------------------\n",
    "\n",
    " cv_method=None : this options is not finished\n",
    "\n",
    " ref_batch= ref_batch: if ref_batch exist, is an integer corresponding to batch id, otherwise ref_batch=None\n",
    "                       In case reference is defined, the LR is only fitted in the reference data.\n",
    "\n",
    " output_dictionary: False -> outputs the corrected data only array (n_subj x n_feat);\n",
    "                    True -> outputs a dictionary of the form of the input with corrected\n",
    "                    data and respective covariates {'data': corrected_output ; 'covariates': covars_matrix}\n",
    "\n",
    " feat_detail: The details of the features to correct which are organized in a dictionary of dictionaries\n",
    "              similarly to the ComBatHarmonization.\n",
    "              Each first level of the dictionary corresponds to a set of features of the same name, e.g: Cortical thickness\n",
    "              , Volumes, surface areas, curvature; for each of this first level a second within dictionary is defined with\n",
    "              the column id of those features in the dataset 'id' and the corresponding biocovariates to remove\n",
    "              in case there is none, define 'categorical' or 'continuous' with a empty list (eg: 'categorical': [] )\n",
    "\n",
    "              dict= {'name_of_feature_type_1: {'id': columns_id_corresponding,\n",
    "              'categorical': list_of_names_of_column_in_dataframe_of_categorical_covaariates,\n",
    "              'continuous': list_of_names_of_column_in_dataframe_of_continuous_covaariates }\n",
    "\n",
    "              -each of the levels of feature_type_i uses different biocovariates in the LR;\n",
    "\n",
    "feat_of_no_interest: better not to use this option for now, but it was to give\n",
    "                     the covariates to harmonize before harmonizing the main features.\n",
    "-------Input--------------------------------------------------------------------\n",
    "\n",
    "dictionary_input: a dictionary with 2 keys, 'data' and 'covariates'.\n",
    "                  'data': a dataframe (n_subjxn_feat)\n",
    "                  'covariates' a dataframe (n_subj x n_covariates) -> the columns\n",
    "                  in here need to have the names of biocovariates consistent with\n",
    "                  the ones in feat_detail 'continuous' and 'categorical'. It needs\n",
    "                  also the batch columns with the batch id's as integers! The batch\n",
    "                  columns needs to be named exactly 'batch' (see above strat_covars\n",
    "                  dataframe for a concrete example).\n",
    "------Ouput---------------------------------------------------------------------\n",
    "Depending on the output_dictionary option:\n",
    "\n",
    "output_dictionary: False -> outputs the corrected data only array (n_subj x n_feat);\n",
    "                    True -> outputs a dictionary of the form of the input with corrected\n",
    "                    data and respective covariates {'data': corrected_output ; 'covariates': covars_matrix}\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "biocovariates_linear_regression=BiocovariatesRegression(ref_batch=None,\n",
    "                                                        cv_method=None,\n",
    "                                                        feat_detail=feat_detail,\n",
    "                                                        output_dict=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_Me496NT4Ufd"
   },
   "source": [
    "# Explaining StandardScalerDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sfrdpqJA4ZFe"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "StandardScalerDict is a standardscaler class created to be compatible with\n",
    "ComBatHarmonization and BiocovRegression where we can define a ref_batch to which\n",
    "to fit the scaler and can choose whether to standardiza 'data' and 'covariates'\n",
    "in the dictionary or just 'data' or just 'covariates'\n",
    "\n",
    "------StandardScalerDict Attributes--------------------------------------------\n",
    "\n",
    "\n",
    " ref_batch= ref_batch: if ref_batch exist, is an integer corresponding to batch id, otherwise ref_batch=None\n",
    "                       In case reference is defined, the scaler is only fitted in the reference data.\n",
    "\n",
    " output_dictionary: False -> outputs the corrected data only array (n_subj x n_feat);\n",
    "                    True -> outputs a dictionary of the form of the input with corrected\n",
    "                    data and respective covariates {'data': corrected_output ; 'covariates': covars_matrix}\n",
    "\n",
    " std_data: True/False -> standardize feature data (n_subj x n_feat)\n",
    " std_cov : True/False -> standardize biocovariates (n_subj x n_cov)\n",
    "\n",
    "output_dictionary: False -> outputs the standardize data only array (n_subj x n_feat);\n",
    "                    True -> outputs a dictionary of the form of the input with corrected\n",
    "                    data and respective covariates {'data': output_data ; 'covariates': output_covars}\n",
    "-------Input--------------------------------------------------------------------\n",
    "\n",
    "dictionary_input: a dictionary with 2 keys, 'data' and 'covariates'.\n",
    "                  'data': a dataframe (n_subjxn_feat)\n",
    "                  'covariates' a dataframe (n_subj x n_covariates) -> the columns\n",
    "                  in here need to have the names of biocovariates consistent with\n",
    "                  the ones in feat_detail 'continuous' and 'categorical'. It needs\n",
    "                  also the batch columns with the batch id's as integers! The batch\n",
    "                  columns needs to be named exactly 'batch' (see above strat_covars\n",
    "                  dataframe for a concrete example).\n",
    "------Ouput---------------------------------------------------------------------\n",
    "Depending on the output_dictionary option:\n",
    "\n",
    "output_dictionary: False -> outputs the standardize data only array (n_subj x n_feat);\n",
    "                    True -> outputs a dictionary of the form of the input with corrected\n",
    "                    data and respective covariates {'data': output_data ; 'covariates': output_covars}\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "standard_scaler_function=StandardScalerDict(ref_batch=None,\n",
    "                                            std_data=True,\n",
    "                                            std_cov=False,\n",
    "                                            output_dict=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XLJZ7Ebi6NOi"
   },
   "source": [
    "# Defining a Pipeline with sklearn with the Confounder Removal Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0OlxFX5k6Te1"
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "----- Steps of the Pipeline in orde--------------------------------------------\n",
    "\n",
    "1. ComBat Harmonization -> output dictionary 'data' and 'covariates;\n",
    "2. Standard Scaler -> standardize only 'data'; output dictionary\n",
    "3. BiocovariatesRegression -> output only data ;\n",
    "4. Standard Scaler -> standardized adjusted data ;\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "combat_function=ComBatHarmonization(cv_method=None, ref_batch=None,\n",
    "                                    regression_fit=1,\n",
    "                                    feat_detail=feat_detail,\n",
    "                                    feat_of_no_interest=None)\n",
    "\n",
    "\n",
    "scaler_data_fun_1=StandardScalerDict(ref_batch=None,\n",
    "                                     std_data=True,\n",
    "                                     std_cov=False,\n",
    "                                     output_dict=True)\n",
    "\n",
    "biocov_regre=BiocovariatesRegression(ref_batch=None,\n",
    "                                     cv_method=None,\n",
    "                                     feat_detail=feat_detail,\n",
    "                                     output_dict=True)\n",
    "\n",
    "scaler_data_fun_2=StandardScalerDict(ref_batch=None,\n",
    "                                     std_data=True,\n",
    "                                     std_cov=False,\n",
    "                                     output_dict=False)\n",
    "\n",
    "confounder_removal_steps = [\n",
    "    ('Multi-center features Harmonization', combat_function),\n",
    "    ('Standardize only data 1', scaler_data_fun_1),\n",
    "    ('Linear Confounder Regression', biocov_regre),\n",
    "    ('Standardize only data 2', scaler_data_fun_2)]\n",
    "\n",
    "CR_pipeline = Pipeline(confounder_removal_steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FzhCdF3e70jw"
   },
   "source": [
    "Some changes we need to do to: defining all the data properly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 344,
     "status": "ok",
     "timestamp": 1699866605112,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "OhfgEyYx76GB",
    "outputId": "3020bd91-1d9f-4512-aa70-b794db0d3b77"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1\n",
       "1      1\n",
       "2      1\n",
       "3      1\n",
       "4      1\n",
       "      ..\n",
       "558    7\n",
       "559    7\n",
       "560    7\n",
       "561    7\n",
       "562    7\n",
       "Name: batch, Length: 563, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Strat_data_raw=Strat_data_raw.iloc[:,0:170] # we select only the features we are interested in using\n",
    "strat_covars.rename(columns={\"Batch\":\"batch\"}, inplace=True) # 'Batch' column defined exactly as 'batch'\n",
    "strat_covars['batch'].astype('int') # make sure batch is of type integer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1699866612556,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "OwRwU14y8e_k",
    "outputId": "d30aef53-2ca6-42fa-e47c-4e329eab7fe7"
   },
   "outputs": [],
   "source": [
    "strat_covars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1699866625432,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "DoxqQYc38iBB",
    "outputId": "c5135ecb-aeb7-4be1-86f7-1ee907bca06d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(563, 170)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Strat_data_raw.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wYbvIYz99Nun"
   },
   "source": [
    "# Split in training and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BuVYF9gN9R8f"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "y=strat_covars.Diagnosis.values\n",
    "\n",
    "X_train, X_test, y_train, y_test, train_ind, test_ind = train_test_split(Strat_data_raw, y, np.arange(0,len(Strat_data_raw)), test_size=0.3, random_state=42)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "e6BaIbI8-Aej"
   },
   "outputs": [],
   "source": [
    "covars_train=strat_covars.loc[train_ind,:]\n",
    "covars_test=strat_covars.loc[test_ind, :]\n",
    "globals_train=strat_globals_data.loc[train_ind,:]\n",
    "globals_test=strat_globals_data.loc[test_ind,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 443
    },
    "executionInfo": {
     "elapsed": 242,
     "status": "ok",
     "timestamp": 1699866947462,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "-1E9Btrg9yGr",
    "outputId": "5d91f691-c287-4bfe-b404-ec76a2b5e194"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-c104cea6-9eb6-4028-bcb7-3b70179c65f5\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>160</th>\n",
       "      <th>161</th>\n",
       "      <th>162</th>\n",
       "      <th>163</th>\n",
       "      <th>164</th>\n",
       "      <th>165</th>\n",
       "      <th>166</th>\n",
       "      <th>167</th>\n",
       "      <th>168</th>\n",
       "      <th>169</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>2.552210</td>\n",
       "      <td>2.462990</td>\n",
       "      <td>2.468973</td>\n",
       "      <td>2.461979</td>\n",
       "      <td>2.493731</td>\n",
       "      <td>2.471658</td>\n",
       "      <td>1.900325</td>\n",
       "      <td>1.940967</td>\n",
       "      <td>2.885210</td>\n",
       "      <td>2.989846</td>\n",
       "      <td>...</td>\n",
       "      <td>0.012122</td>\n",
       "      <td>0.087965</td>\n",
       "      <td>0.043639</td>\n",
       "      <td>0.294800</td>\n",
       "      <td>0.005631</td>\n",
       "      <td>0.002643</td>\n",
       "      <td>0.005686</td>\n",
       "      <td>0.037894</td>\n",
       "      <td>0.014294</td>\n",
       "      <td>0.027392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>2.458855</td>\n",
       "      <td>2.617612</td>\n",
       "      <td>2.329191</td>\n",
       "      <td>2.633159</td>\n",
       "      <td>2.589748</td>\n",
       "      <td>2.545012</td>\n",
       "      <td>2.100493</td>\n",
       "      <td>2.080373</td>\n",
       "      <td>3.232171</td>\n",
       "      <td>3.514806</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023824</td>\n",
       "      <td>0.074412</td>\n",
       "      <td>0.032000</td>\n",
       "      <td>0.243329</td>\n",
       "      <td>0.006694</td>\n",
       "      <td>0.003878</td>\n",
       "      <td>0.004631</td>\n",
       "      <td>0.036616</td>\n",
       "      <td>0.020204</td>\n",
       "      <td>0.010506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>2.511254</td>\n",
       "      <td>2.609719</td>\n",
       "      <td>2.308973</td>\n",
       "      <td>2.476828</td>\n",
       "      <td>2.496947</td>\n",
       "      <td>2.496931</td>\n",
       "      <td>1.934517</td>\n",
       "      <td>1.969216</td>\n",
       "      <td>3.737791</td>\n",
       "      <td>3.779139</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008404</td>\n",
       "      <td>0.055043</td>\n",
       "      <td>0.033408</td>\n",
       "      <td>0.203851</td>\n",
       "      <td>0.014996</td>\n",
       "      <td>0.008627</td>\n",
       "      <td>0.013231</td>\n",
       "      <td>0.027776</td>\n",
       "      <td>0.020039</td>\n",
       "      <td>0.026871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.572397</td>\n",
       "      <td>2.452712</td>\n",
       "      <td>2.432269</td>\n",
       "      <td>2.509864</td>\n",
       "      <td>2.544981</td>\n",
       "      <td>2.699870</td>\n",
       "      <td>1.811273</td>\n",
       "      <td>1.848244</td>\n",
       "      <td>4.133728</td>\n",
       "      <td>3.841347</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015298</td>\n",
       "      <td>0.049278</td>\n",
       "      <td>0.039478</td>\n",
       "      <td>0.266698</td>\n",
       "      <td>0.003647</td>\n",
       "      <td>0.001682</td>\n",
       "      <td>0.004216</td>\n",
       "      <td>0.027153</td>\n",
       "      <td>0.016067</td>\n",
       "      <td>0.002490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>2.611566</td>\n",
       "      <td>2.730710</td>\n",
       "      <td>2.388031</td>\n",
       "      <td>2.508102</td>\n",
       "      <td>2.650707</td>\n",
       "      <td>2.622227</td>\n",
       "      <td>1.829096</td>\n",
       "      <td>1.879593</td>\n",
       "      <td>3.758057</td>\n",
       "      <td>3.448658</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002499</td>\n",
       "      <td>0.024971</td>\n",
       "      <td>0.011185</td>\n",
       "      <td>0.304096</td>\n",
       "      <td>0.000311</td>\n",
       "      <td>0.009438</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.024995</td>\n",
       "      <td>0.014315</td>\n",
       "      <td>0.011022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>2.241153</td>\n",
       "      <td>2.510218</td>\n",
       "      <td>2.862993</td>\n",
       "      <td>2.203013</td>\n",
       "      <td>2.435800</td>\n",
       "      <td>2.573043</td>\n",
       "      <td>1.840058</td>\n",
       "      <td>2.024040</td>\n",
       "      <td>3.397003</td>\n",
       "      <td>3.713032</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022349</td>\n",
       "      <td>0.083733</td>\n",
       "      <td>0.025400</td>\n",
       "      <td>0.311318</td>\n",
       "      <td>0.012839</td>\n",
       "      <td>0.005369</td>\n",
       "      <td>0.002329</td>\n",
       "      <td>0.044753</td>\n",
       "      <td>0.023357</td>\n",
       "      <td>0.006447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>2.483150</td>\n",
       "      <td>2.471641</td>\n",
       "      <td>2.116892</td>\n",
       "      <td>2.348220</td>\n",
       "      <td>2.500250</td>\n",
       "      <td>2.439027</td>\n",
       "      <td>1.829013</td>\n",
       "      <td>1.830314</td>\n",
       "      <td>3.650863</td>\n",
       "      <td>4.193119</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015582</td>\n",
       "      <td>0.034486</td>\n",
       "      <td>0.023354</td>\n",
       "      <td>0.288929</td>\n",
       "      <td>0.000533</td>\n",
       "      <td>0.012000</td>\n",
       "      <td>0.004973</td>\n",
       "      <td>0.039170</td>\n",
       "      <td>0.025119</td>\n",
       "      <td>0.016278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>2.336187</td>\n",
       "      <td>2.644528</td>\n",
       "      <td>2.482437</td>\n",
       "      <td>2.411128</td>\n",
       "      <td>2.447669</td>\n",
       "      <td>2.350911</td>\n",
       "      <td>1.939114</td>\n",
       "      <td>2.034813</td>\n",
       "      <td>4.014911</td>\n",
       "      <td>3.871373</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013761</td>\n",
       "      <td>0.090071</td>\n",
       "      <td>0.058145</td>\n",
       "      <td>0.237988</td>\n",
       "      <td>0.005792</td>\n",
       "      <td>0.000435</td>\n",
       "      <td>0.011173</td>\n",
       "      <td>0.020741</td>\n",
       "      <td>0.026863</td>\n",
       "      <td>0.035761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>2.275367</td>\n",
       "      <td>2.302618</td>\n",
       "      <td>2.284392</td>\n",
       "      <td>2.386352</td>\n",
       "      <td>1.830458</td>\n",
       "      <td>1.934811</td>\n",
       "      <td>1.590133</td>\n",
       "      <td>1.329453</td>\n",
       "      <td>3.093001</td>\n",
       "      <td>3.400304</td>\n",
       "      <td>...</td>\n",
       "      <td>0.017161</td>\n",
       "      <td>0.043275</td>\n",
       "      <td>0.038310</td>\n",
       "      <td>0.244282</td>\n",
       "      <td>0.013027</td>\n",
       "      <td>0.017671</td>\n",
       "      <td>0.002298</td>\n",
       "      <td>0.035847</td>\n",
       "      <td>0.025808</td>\n",
       "      <td>0.013957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>2.487143</td>\n",
       "      <td>2.497343</td>\n",
       "      <td>2.362207</td>\n",
       "      <td>2.328506</td>\n",
       "      <td>2.661500</td>\n",
       "      <td>2.509439</td>\n",
       "      <td>1.720261</td>\n",
       "      <td>1.894676</td>\n",
       "      <td>2.908740</td>\n",
       "      <td>3.080924</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018600</td>\n",
       "      <td>0.052659</td>\n",
       "      <td>0.038234</td>\n",
       "      <td>0.273301</td>\n",
       "      <td>0.015932</td>\n",
       "      <td>0.011051</td>\n",
       "      <td>0.000653</td>\n",
       "      <td>0.014153</td>\n",
       "      <td>0.031137</td>\n",
       "      <td>0.009720</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394 rows × 170 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c104cea6-9eb6-4028-bcb7-3b70179c65f5')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-c104cea6-9eb6-4028-bcb7-3b70179c65f5 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-c104cea6-9eb6-4028-bcb7-3b70179c65f5');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-49bfb027-8a61-4e45-b481-fff0ada6cc9b\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-49bfb027-8a61-4e45-b481-fff0ada6cc9b')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-49bfb027-8a61-4e45-b481-fff0ada6cc9b button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "            0         1         2         3         4         5         6  \\\n",
       "298  2.552210  2.462990  2.468973  2.461979  2.493731  2.471658  1.900325   \n",
       "536  2.458855  2.617612  2.329191  2.633159  2.589748  2.545012  2.100493   \n",
       "464  2.511254  2.609719  2.308973  2.476828  2.496947  2.496931  1.934517   \n",
       "19   2.572397  2.452712  2.432269  2.509864  2.544981  2.699870  1.811273   \n",
       "172  2.611566  2.730710  2.388031  2.508102  2.650707  2.622227  1.829096   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "71   2.241153  2.510218  2.862993  2.203013  2.435800  2.573043  1.840058   \n",
       "106  2.483150  2.471641  2.116892  2.348220  2.500250  2.439027  1.829013   \n",
       "270  2.336187  2.644528  2.482437  2.411128  2.447669  2.350911  1.939114   \n",
       "435  2.275367  2.302618  2.284392  2.386352  1.830458  1.934811  1.590133   \n",
       "102  2.487143  2.497343  2.362207  2.328506  2.661500  2.509439  1.720261   \n",
       "\n",
       "            7         8         9  ...       160       161       162  \\\n",
       "298  1.940967  2.885210  2.989846  ...  0.012122  0.087965  0.043639   \n",
       "536  2.080373  3.232171  3.514806  ...  0.023824  0.074412  0.032000   \n",
       "464  1.969216  3.737791  3.779139  ...  0.008404  0.055043  0.033408   \n",
       "19   1.848244  4.133728  3.841347  ...  0.015298  0.049278  0.039478   \n",
       "172  1.879593  3.758057  3.448658  ...  0.002499  0.024971  0.011185   \n",
       "..        ...       ...       ...  ...       ...       ...       ...   \n",
       "71   2.024040  3.397003  3.713032  ...  0.022349  0.083733  0.025400   \n",
       "106  1.830314  3.650863  4.193119  ...  0.015582  0.034486  0.023354   \n",
       "270  2.034813  4.014911  3.871373  ...  0.013761  0.090071  0.058145   \n",
       "435  1.329453  3.093001  3.400304  ...  0.017161  0.043275  0.038310   \n",
       "102  1.894676  2.908740  3.080924  ...  0.018600  0.052659  0.038234   \n",
       "\n",
       "          163       164       165       166       167       168       169  \n",
       "298  0.294800  0.005631  0.002643  0.005686  0.037894  0.014294  0.027392  \n",
       "536  0.243329  0.006694  0.003878  0.004631  0.036616  0.020204  0.010506  \n",
       "464  0.203851  0.014996  0.008627  0.013231  0.027776  0.020039  0.026871  \n",
       "19   0.266698  0.003647  0.001682  0.004216  0.027153  0.016067  0.002490  \n",
       "172  0.304096  0.000311  0.009438  0.000000  0.024995  0.014315  0.011022  \n",
       "..        ...       ...       ...       ...       ...       ...       ...  \n",
       "71   0.311318  0.012839  0.005369  0.002329  0.044753  0.023357  0.006447  \n",
       "106  0.288929  0.000533  0.012000  0.004973  0.039170  0.025119  0.016278  \n",
       "270  0.237988  0.005792  0.000435  0.011173  0.020741  0.026863  0.035761  \n",
       "435  0.244282  0.013027  0.017671  0.002298  0.035847  0.025808  0.013957  \n",
       "102  0.273301  0.015932  0.011051  0.000653  0.014153  0.031137  0.009720  \n",
       "\n",
       "[394 rows x 170 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1699867110011,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "vV9Tn6E4-Zzs",
    "outputId": "1c449fab-91fe-4eae-dbfe-0412da1b32ad"
   },
   "outputs": [],
   "source": [
    "covars_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 304,
     "status": "ok",
     "timestamp": 1699867128417,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "nB9txqgh-cor",
    "outputId": "c7a46dc8-4ea7-4bb8-b6a3-12f372e5bed1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-2fdb839b-5a39-46b1-986a-e7fdc5aa1fc3\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIV</th>\n",
       "      <th>CT_avg</th>\n",
       "      <th>CT_std</th>\n",
       "      <th>abGM</th>\n",
       "      <th>abWM</th>\n",
       "      <th>abCSF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>1574.660548</td>\n",
       "      <td>2.351142</td>\n",
       "      <td>0.640188</td>\n",
       "      <td>719.254309</td>\n",
       "      <td>557.735553</td>\n",
       "      <td>297.670686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>1406.606451</td>\n",
       "      <td>2.458025</td>\n",
       "      <td>0.621959</td>\n",
       "      <td>676.723698</td>\n",
       "      <td>474.892451</td>\n",
       "      <td>254.990302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>1405.013503</td>\n",
       "      <td>2.394365</td>\n",
       "      <td>0.596628</td>\n",
       "      <td>649.976389</td>\n",
       "      <td>480.497547</td>\n",
       "      <td>274.539567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1374.348110</td>\n",
       "      <td>2.393596</td>\n",
       "      <td>0.630534</td>\n",
       "      <td>628.093816</td>\n",
       "      <td>499.739141</td>\n",
       "      <td>246.515153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>1251.392120</td>\n",
       "      <td>2.436498</td>\n",
       "      <td>0.650705</td>\n",
       "      <td>587.247112</td>\n",
       "      <td>441.656601</td>\n",
       "      <td>222.488407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>1443.336396</td>\n",
       "      <td>2.343455</td>\n",
       "      <td>0.639852</td>\n",
       "      <td>641.887408</td>\n",
       "      <td>537.210227</td>\n",
       "      <td>264.238761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>1389.857416</td>\n",
       "      <td>2.355177</td>\n",
       "      <td>0.646240</td>\n",
       "      <td>591.756621</td>\n",
       "      <td>491.540049</td>\n",
       "      <td>306.560747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>1625.616049</td>\n",
       "      <td>2.402215</td>\n",
       "      <td>0.607347</td>\n",
       "      <td>683.286987</td>\n",
       "      <td>573.735825</td>\n",
       "      <td>368.593236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>1320.458580</td>\n",
       "      <td>2.028008</td>\n",
       "      <td>0.690220</td>\n",
       "      <td>495.833629</td>\n",
       "      <td>530.602392</td>\n",
       "      <td>294.022559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>1399.300491</td>\n",
       "      <td>2.342054</td>\n",
       "      <td>0.635299</td>\n",
       "      <td>555.797511</td>\n",
       "      <td>515.279283</td>\n",
       "      <td>328.223698</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394 rows × 6 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2fdb839b-5a39-46b1-986a-e7fdc5aa1fc3')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-2fdb839b-5a39-46b1-986a-e7fdc5aa1fc3 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-2fdb839b-5a39-46b1-986a-e7fdc5aa1fc3');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-9d544b35-a6f7-4d3c-acd6-96e7a0bce81b\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9d544b35-a6f7-4d3c-acd6-96e7a0bce81b')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-9d544b35-a6f7-4d3c-acd6-96e7a0bce81b button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "             TIV    CT_avg    CT_std        abGM        abWM       abCSF\n",
       "298  1574.660548  2.351142  0.640188  719.254309  557.735553  297.670686\n",
       "536  1406.606451  2.458025  0.621959  676.723698  474.892451  254.990302\n",
       "464  1405.013503  2.394365  0.596628  649.976389  480.497547  274.539567\n",
       "19   1374.348110  2.393596  0.630534  628.093816  499.739141  246.515153\n",
       "172  1251.392120  2.436498  0.650705  587.247112  441.656601  222.488407\n",
       "..           ...       ...       ...         ...         ...         ...\n",
       "71   1443.336396  2.343455  0.639852  641.887408  537.210227  264.238761\n",
       "106  1389.857416  2.355177  0.646240  591.756621  491.540049  306.560747\n",
       "270  1625.616049  2.402215  0.607347  683.286987  573.735825  368.593236\n",
       "435  1320.458580  2.028008  0.690220  495.833629  530.602392  294.022559\n",
       "102  1399.300491  2.342054  0.635299  555.797511  515.279283  328.223698\n",
       "\n",
       "[394 rows x 6 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "globals_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qMgFWiPeIv0P"
   },
   "source": [
    "# Harmonize Total Intracranial Volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 255,
     "status": "ok",
     "timestamp": 1699867516844,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "u26uJIdx7QTW",
    "outputId": "0dcb7820-0bff-4db5-99b4-171c2f926192"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3 4 5 6 7]\n",
      "[neuroCombat] Creating design matrix\n",
      "[neuroCombat] Standardizing data across features\n",
      "[neuroCombat] Fitting L/S model and finding priors\n",
      "[neuroCombat] Finding parametric adjustments\n",
      "[neuroCombat] Final adjustment of data\n",
      "Batchs for this set ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batchs from the training set estimation ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batches in current set matching the ones from training set estimation [0 1 2 3 4 5 6]\n",
      "Transforming a training set\n",
      "Batchs for this set ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batchs from the training set estimation ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batches in current set matching the ones from training set estimation [0 1 2 3 4 5 6]\n",
      "Transforming a test set\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# M-ComBat: If there is a reference set : ref_batch\n",
    "\n",
    "globals_id=np.arange(0,6)\n",
    "\n",
    "# The TIV is harmonized together with globals\n",
    "globals_of_interest={'globals': {'id': globals_id, 'categorical': ['Gender'],\n",
    "                               'continuous':['Age']} }\n",
    "\n",
    "# Define combat object\n",
    "combat_function_TIV=ComBatHarmonization(cv_method=None, ref_batch=None,\n",
    "                                        regression_fit=0,\n",
    "                                        feat_detail=globals_of_interest,\n",
    "                                        feat_of_no_interest=None)\n",
    "\n",
    "# Harmonize TIV\n",
    "\n",
    "# Define input\n",
    "train_globals_dict={'data': globals_train, 'covariates': covars_train}\n",
    "\n",
    "# Harmonize Training Globals Data: fit and transform\n",
    "globals_train_harm=combat_function_TIV.fit_transform(train_globals_dict)\n",
    "\n",
    "# Harmonize Testing Globals Data: fit and transform\n",
    "\n",
    "test_globals_dict={'data': globals_test, 'covariates': covars_test}\n",
    "\n",
    "globals_test_harm=combat_function_TIV.transform(test_globals_dict)\n",
    "\n",
    "# Extrat harmonized TIV\n",
    "train_harm_TIV=pd.DataFrame(globals_train_harm[:,0], columns=['TIV'])\n",
    "test_harm_TIV=pd.DataFrame(globals_test_harm[:,0], columns=['TIV'])\n",
    "\n",
    "# Replace raw TIV covariate with harmonized TIV\n",
    "covars_train['TIV']=globals_train_harm[:,0]\n",
    "covars_test['TIV']=globals_test_harm[:,0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1699867537298,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "qamc7oDLACbx",
    "outputId": "93ff5823-9f22-4ca9-c259-b4a9bd8a8ec1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.54713715e+03, 2.29884741e+00, 6.47338524e-01, 6.93333398e+02,\n",
       "        5.62455132e+02, 2.91572144e+02],\n",
       "       [1.34509789e+03, 2.35307863e+00, 6.14414476e-01, 6.07365925e+02,\n",
       "        4.57617208e+02, 2.76527746e+02],\n",
       "       [1.43141268e+03, 2.32056345e+00, 6.10972065e-01, 6.32488130e+02,\n",
       "        5.03350281e+02, 2.94744371e+02],\n",
       "       ...,\n",
       "       [1.59754656e+03, 2.35154965e+00, 6.14159144e-01, 6.58789784e+02,\n",
       "        5.78799469e+02, 3.63457368e+02],\n",
       "       [1.38854082e+03, 2.26183965e+00, 6.50917221e-01, 5.76299249e+02,\n",
       "        5.22316565e+02, 2.97428690e+02],\n",
       "       [1.38972041e+03, 2.35150730e+00, 6.44974050e-01, 5.77222299e+02,\n",
       "        5.18050107e+02, 2.89067441e+02]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "globals_train_harm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 245,
     "status": "ok",
     "timestamp": 1699867556151,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "7BNJRT5EAEiu",
    "outputId": "5e00d6f3-b260-4a79-d799-5ea7711a6837"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-fed25727-168e-44cb-b0d8-80c85bb6133f\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>1574.660548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>1406.606451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>1405.013503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1374.348110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>1251.392120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>1443.336396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>1389.857416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>1625.616049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>1320.458580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>1399.300491</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394 rows × 1 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fed25727-168e-44cb-b0d8-80c85bb6133f')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-fed25727-168e-44cb-b0d8-80c85bb6133f button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-fed25727-168e-44cb-b0d8-80c85bb6133f');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-bfe052e8-e787-41ad-9975-476238d628ea\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-bfe052e8-e787-41ad-9975-476238d628ea')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-bfe052e8-e787-41ad-9975-476238d628ea button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "             TIV\n",
       "298  1574.660548\n",
       "536  1406.606451\n",
       "464  1405.013503\n",
       "19   1374.348110\n",
       "172  1251.392120\n",
       "..           ...\n",
       "71   1443.336396\n",
       "106  1389.857416\n",
       "270  1625.616049\n",
       "435  1320.458580\n",
       "102  1399.300491\n",
       "\n",
       "[394 rows x 1 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Original TIV\n",
    "globals_train[['TIV']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1699867527383,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "ATRtd-D7_-uk",
    "outputId": "0af83d42-10f9-4eb2-d874-ad8dfc64307c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-49b8ee1f-5187-4f7e-bb9d-7d134bfb68cf\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TIV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1547.137147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1345.097891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1431.412678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1375.340296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1240.227421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>1463.073044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>390</th>\n",
       "      <td>1380.785219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391</th>\n",
       "      <td>1597.546561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>1388.540821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>1389.720405</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394 rows × 1 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-49b8ee1f-5187-4f7e-bb9d-7d134bfb68cf')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-49b8ee1f-5187-4f7e-bb9d-7d134bfb68cf button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-49b8ee1f-5187-4f7e-bb9d-7d134bfb68cf');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-9e7237b6-d5d7-464e-8f56-38f87db73940\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-9e7237b6-d5d7-464e-8f56-38f87db73940')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-9e7237b6-d5d7-464e-8f56-38f87db73940 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "             TIV\n",
       "0    1547.137147\n",
       "1    1345.097891\n",
       "2    1431.412678\n",
       "3    1375.340296\n",
       "4    1240.227421\n",
       "..           ...\n",
       "389  1463.073044\n",
       "390  1380.785219\n",
       "391  1597.546561\n",
       "392  1388.540821\n",
       "393  1389.720405\n",
       "\n",
       "[394 rows x 1 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Harmonized TIV\n",
    "train_harm_TIV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 230,
     "status": "ok",
     "timestamp": 1699869728287,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "gKSRo6T9IN7R",
    "outputId": "dee4ec7f-00df-452a-b046-527531ba259e"
   },
   "outputs": [],
   "source": [
    "covars_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 423
    },
    "executionInfo": {
     "elapsed": 225,
     "status": "ok",
     "timestamp": 1699869815423,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "Aj1PzCRDIt6u",
    "outputId": "a527ca08-0989-4638-ac1d-b54ed1858bb2"
   },
   "outputs": [],
   "source": [
    "covars_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Lamaj7eh2nkG"
   },
   "source": [
    "# Harmonize Brain Morphological Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 594,
     "status": "ok",
     "timestamp": 1699870194197,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "EV-r9hY02GUx",
    "outputId": "31886a66-be17-474e-ceef-93cc7f9a9d7e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3 4 5 6 7]\n",
      "[neuroCombat] Creating design matrix\n",
      "[neuroCombat] Standardizing data across features\n",
      "[neuroCombat] Fitting L/S model and finding priors\n",
      "[neuroCombat] Finding parametric adjustments\n",
      "[neuroCombat] Final adjustment of data\n",
      "[1 2 3 4 5 6 7]\n",
      "[neuroCombat] Creating design matrix\n",
      "[neuroCombat] Standardizing data across features\n",
      "[neuroCombat] Fitting L/S model and finding priors\n",
      "[neuroCombat] Finding parametric adjustments\n",
      "[neuroCombat] Final adjustment of data\n",
      "Batchs for this set ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batchs from the training set estimation ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batches in current set matching the ones from training set estimation [0 1 2 3 4 5 6]\n",
      "Transforming a training set\n",
      "Batchs for this set ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batchs from the training set estimation ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batches in current set matching the ones from training set estimation [0 1 2 3 4 5 6]\n",
      "Transforming a training set\n",
      "SS: Shape of the whole dataset (394, 170)\n",
      "SS: Transforming dataset shape (394, 170)\n",
      "LR: Shape of the whole dataset (394, 170)\n",
      "cortical\n",
      "volumes\n",
      "LR: Transforming dataset shape (394, 170)\n",
      "SS: Shape of the whole dataset (394, 170)\n",
      "SS: Transforming dataset shape (394, 170)\n",
      "Batchs for this set ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batchs from the training set estimation ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batches in current set matching the ones from training set estimation [0 1 2 3 4 5 6]\n",
      "Transforming a test set\n",
      "Batchs for this set ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batchs from the training set estimation ['1' '2' '3' '4' '5' '6' '7']\n",
      "Batches in current set matching the ones from training set estimation [0 1 2 3 4 5 6]\n",
      "Transforming a test set\n",
      "SS: Transforming dataset shape (169, 170)\n",
      "LR: Transforming dataset shape (169, 170)\n",
      "SS: Transforming dataset shape (169, 170)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Define inputs\n",
    "train_strat_dict={'data': X_train, 'covariates': covars_train}\n",
    "test_strat_dict={'data': X_test, 'covariates': covars_test}\n",
    "\n",
    "# Brain feature-types to consider separatly and associated bio-covariates\n",
    "features_info={'cortical': {'id': cortical_columns,\n",
    "                               'categorical': ['Gender'],\n",
    "                               'continuous':['Age']},\n",
    "                  'volumes': {'id': volumes_columns,\n",
    "                              'categorical': ['Gender'],\n",
    "                              'continuous':['Age','TIV']}}\n",
    "\n",
    "# Confounder Removal Pipelne\n",
    "\n",
    "combat_data=ComBatHarmonization(cv_method=None, ref_batch=None,\n",
    "                                    regression_fit=1,\n",
    "                                    feat_detail=features_info,\n",
    "                                    feat_of_no_interest=None)\n",
    "\n",
    "scaler_data_fun_1=StandardScalerDict(ref_batch=None,std_data=True,\n",
    "                                     std_cov=False,\n",
    "                                     output_dict=True)\n",
    "\n",
    "biocov_regre=BiocovariatesRegression(ref_batch=None,cv_method=None,\n",
    "                                     feat_detail=features_info,\n",
    "                                     output_dict=True)\n",
    "\n",
    "scaler_data_fun_2=StandardScalerDict(ref_batch=None,std_data=True,\n",
    "                                     std_cov=False,\n",
    "                                     output_dict=False)\n",
    "\n",
    "combat_biocovregr_steps = [('Multi-center features Harmonization', combat_data),\n",
    " ('Standardize only data 1', scaler_data_fun_1),('Confounder Regression', biocov_regre),\n",
    "  ('Standardize only data 2', scaler_data_fun_2)]\n",
    "\n",
    "preprocessing_pipeline = Pipeline(combat_biocovregr_steps)\n",
    "\n",
    "# Fit_transform method in training set\n",
    "X_train_corr=preprocessing_pipeline.fit_transform(train_strat_dict)\n",
    "\n",
    "# Transforming Test set\n",
    "X_test_corr=preprocessing_pipeline.transform(test_strat_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1699870208530,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "vshLl7otKI2v",
    "outputId": "322086e3-f31a-4d96-c533-2c6d4551c04b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.41930519, -0.92971103, -0.33455092, ...,  1.20128136,\n",
       "        -0.92153939,  1.4000994 ],\n",
       "       [-0.39144888, -0.13532221, -0.2985827 , ...,  0.44821857,\n",
       "         0.645937  , -0.40822629],\n",
       "       [ 0.07515839,  0.12569344, -0.32469878, ..., -0.40159271,\n",
       "         0.06793052,  1.1883009 ],\n",
       "       ...,\n",
       "       [-0.91631291,  0.21893407, -0.32608107, ..., -0.79126362,\n",
       "         1.0319526 ,  1.86557707],\n",
       "       [-0.09518649, -0.10600578, -0.64283027, ...,  1.19386652,\n",
       "         1.0735958 , -0.42357044],\n",
       "       [ 0.1495712 , -0.28161604, -0.10774818, ..., -1.86401587,\n",
       "         1.46778723, -0.77247772]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4,
     "status": "ok",
     "timestamp": 1699870214824,
     "user": {
      "displayName": "inês Sampaio",
      "userId": "15878877488891250619"
     },
     "user_tz": -60
    },
    "id": "KglGBrgVKPBX",
    "outputId": "b1e8cf98-6a6f-4afc-e55e-794733a6712a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.08925469,  1.84780032,  0.96655998, ...,  0.25388786,\n",
       "        -1.29767768, -0.18339876],\n",
       "       [ 0.15864629, -0.56302749,  0.25121383, ...,  0.81308577,\n",
       "        -0.57991426, -1.41909502],\n",
       "       [ 1.42306859, -0.90340363,  1.50549676, ...,  0.21577087,\n",
       "        -0.47745323, -1.02438088],\n",
       "       ...,\n",
       "       [ 1.35054951,  0.24282036,  1.66479713, ...,  0.53370873,\n",
       "         0.34335469, -0.92056434],\n",
       "       [ 1.69328459,  1.28252254,  2.44082106, ..., -0.76028102,\n",
       "        -1.2045755 , -0.18999572],\n",
       "       [ 0.20905805, -0.18965182, -1.4106118 , ...,  0.8651249 ,\n",
       "         1.17539661,  0.06137192]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "u__q51fHKQjz"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}