pimakgcreation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PIMA Knowledge Graph Creation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing and Data Aggregation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set variables according to docker instance\n",
    "uri = \"bolt://localhost:7687\"\n",
    "username = \"neo4j\"\n",
    "password = \"password\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#connect with driver \n",
    "from neo4j import GraphDatabase\n",
    "with GraphDatabase.driver(uri,auth=(username,password)) as driver:\n",
    "  driver.verify_connectivity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup GDS\n",
    "from re import U\n",
    "from graphdatascience import GraphDataScience\n",
    "\n",
    "gds = GraphDataScience(uri, auth=(username,password))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set variables for projected graph embedding projection\n",
    "graph_name = \"pimaGraph\"\n",
    "embedding_dimension = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#project graph with all content under the specified graph_name\n",
    "driver.execute_query(\n",
    "    \"\"\"\n",
    "    CALL gds.graph.project(\n",
    "        $graphName,\n",
    "        {\n",
    "            Sample: '*',\n",
    "            medical_concept: '*',\n",
    "            definition: '*',\n",
    "            synonyms: '*'\n",
    "        },\n",
    "        {\n",
    "            HAS_PREGNANCIES: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            }, \n",
    "            HAS_GLUCOSE: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            HAS_BLOOD_PRESSURE: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            HAS_SKIN_THICKNESS: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            HAS_INSULIN: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            }, \n",
    "            HAS_BMI: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            HAS_DIABETES_PEDIGREE_FUNCTION: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            HAS_AGE: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            }, \n",
    "            Definition : {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            Synonym: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            },\n",
    "            embedding_match_node: {\n",
    "            orientation: 'UNDIRECTED'  \n",
    "            }\n",
    "        })\n",
    "    \"\"\",\n",
    "    graphName=graph_name\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get fastrp embeddings from projected graph with specified embeddingDimension\n",
    "results = driver.execute_query(\n",
    "    \"\"\"\n",
    "    CALL gds.fastRP.stream($graphName, {\n",
    "        embeddingDimension: $embeddingDimension,\n",
    "        randomSeed:42\n",
    "    })\n",
    "    YIELD nodeId, embedding\n",
    "    RETURN nodeId, embedding;\n",
    "    \"\"\",\n",
    "    embeddingDimension=embedding_dimension, graphName=graph_name\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert results into pandas dataframe\n",
    "embedding_df = pd.DataFrame([{\"nodeId\": record[\"nodeId\"], \"embedding\": record[\"embedding\"]} for record in results[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nodeId</th>\n",
       "      <th>embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1157</td>\n",
       "      <td>[0.3707543611526489, 0.054082226008176804, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1159</td>\n",
       "      <td>[0.3539871275424957, 0.05613316223025322, -0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>[0.4141680598258972, 0.03869418054819107, -0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>[0.371269166469574, 0.05201292037963867, -0.10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>[0.3478549122810364, 0.06399303674697876, -0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1152</th>\n",
       "      <td>1150</td>\n",
       "      <td>[0.3686221241950989, 0.026663072407245636, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1153</th>\n",
       "      <td>1151</td>\n",
       "      <td>[0.3833306133747101, 0.05250994861125946, -0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>1152</td>\n",
       "      <td>[0.4162055253982544, 0.026111219078302383, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1155</th>\n",
       "      <td>1153</td>\n",
       "      <td>[0.34819433093070984, 0.0923713967204094, 0.03...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1156</th>\n",
       "      <td>1156</td>\n",
       "      <td>[0.4478495121002197, 0.06772170960903168, -0.0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1157 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      nodeId                                          embedding\n",
       "0       1157  [0.3707543611526489, 0.054082226008176804, -0....\n",
       "1       1159  [0.3539871275424957, 0.05613316223025322, -0.1...\n",
       "2          0  [0.4141680598258972, 0.03869418054819107, -0.1...\n",
       "3          1  [0.371269166469574, 0.05201292037963867, -0.10...\n",
       "4          2  [0.3478549122810364, 0.06399303674697876, -0.1...\n",
       "...      ...                                                ...\n",
       "1152    1150  [0.3686221241950989, 0.026663072407245636, -0....\n",
       "1153    1151  [0.3833306133747101, 0.05250994861125946, -0.0...\n",
       "1154    1152  [0.4162055253982544, 0.026111219078302383, -0....\n",
       "1155    1153  [0.34819433093070984, 0.0923713967204094, 0.03...\n",
       "1156    1156  [0.4478495121002197, 0.06772170960903168, -0.0...\n",
       "\n",
       "[1157 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view dataframe\n",
    "embedding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#recieve Samples and their corresponding ids (so we can get the ids and their respective embeddings)\n",
    "results = driver.execute_query(\n",
    "    \"\"\"\n",
    "    match (m:Sample) return  id(m), m.SampleNumber\n",
    "    \"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert samples into df \n",
    "sample_df = pd.DataFrame([{\"nodeId\": record[\"id(m)\"], \"sampleNumber\": record[\"m.SampleNumber\"]} for record in results[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nodeId</th>\n",
       "      <th>sampleNumber</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>1152</td>\n",
       "      <td>377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>1153</td>\n",
       "      <td>378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>1156</td>\n",
       "      <td>379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>1157</td>\n",
       "      <td>380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>1159</td>\n",
       "      <td>381</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>768 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     nodeId  sampleNumber\n",
       "0         0           382\n",
       "1         1           383\n",
       "2         2           384\n",
       "3         3           385\n",
       "4         4           386\n",
       "..      ...           ...\n",
       "763    1152           377\n",
       "764    1153           378\n",
       "765    1156           379\n",
       "766    1157           380\n",
       "767    1159           381\n",
       "\n",
       "[768 rows x 2 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view df\n",
    "sample_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#merge dataframes\n",
    "combined_df= sample_df.merge(embedding_df, on=\"nodeId\", how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nodeId</th>\n",
       "      <th>sampleNumber</th>\n",
       "      <th>embedding</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>382</td>\n",
       "      <td>[0.4141680598258972, 0.03869418054819107, -0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>383</td>\n",
       "      <td>[0.371269166469574, 0.05201292037963867, -0.10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>384</td>\n",
       "      <td>[0.3478549122810364, 0.06399303674697876, -0.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>385</td>\n",
       "      <td>[0.4099091589450836, 0.027811255306005478, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>386</td>\n",
       "      <td>[0.3573867976665497, 0.0758630633354187, 0.027...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>1152</td>\n",
       "      <td>377</td>\n",
       "      <td>[0.4162055253982544, 0.026111219078302383, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>1153</td>\n",
       "      <td>378</td>\n",
       "      <td>[0.34819433093070984, 0.0923713967204094, 0.03...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>1156</td>\n",
       "      <td>379</td>\n",
       "      <td>[0.4478495121002197, 0.06772170960903168, -0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>1157</td>\n",
       "      <td>380</td>\n",
       "      <td>[0.3707543611526489, 0.054082226008176804, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>1159</td>\n",
       "      <td>381</td>\n",
       "      <td>[0.3539871275424957, 0.05613316223025322, -0.1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>768 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     nodeId  sampleNumber                                          embedding\n",
       "0         0           382  [0.4141680598258972, 0.03869418054819107, -0.1...\n",
       "1         1           383  [0.371269166469574, 0.05201292037963867, -0.10...\n",
       "2         2           384  [0.3478549122810364, 0.06399303674697876, -0.1...\n",
       "3         3           385  [0.4099091589450836, 0.027811255306005478, -0....\n",
       "4         4           386  [0.3573867976665497, 0.0758630633354187, 0.027...\n",
       "..      ...           ...                                                ...\n",
       "763    1152           377  [0.4162055253982544, 0.026111219078302383, -0....\n",
       "764    1153           378  [0.34819433093070984, 0.0923713967204094, 0.03...\n",
       "765    1156           379  [0.4478495121002197, 0.06772170960903168, -0.0...\n",
       "766    1157           380  [0.3707543611526489, 0.054082226008176804, -0....\n",
       "767    1159           381  [0.3539871275424957, 0.05613316223025322, -0.1...\n",
       "\n",
       "[768 rows x 3 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view dataframe\n",
    "combined_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load bucketed dataset \n",
    "diabetes_df = pd.read_csv(\"data/diabetes_final.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SampleNumber</th>\n",
       "      <th>Outcome</th>\n",
       "      <th>Pregnancies_Bucket</th>\n",
       "      <th>Glucose_Bucket</th>\n",
       "      <th>BloodPressure_Bucket</th>\n",
       "      <th>SkinThickness_Bucket</th>\n",
       "      <th>Insulin_Bucket</th>\n",
       "      <th>BMI_Bucket</th>\n",
       "      <th>DiabetesPedigreeFunction_Bucket</th>\n",
       "      <th>Age_Bucket</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>normal pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>high skin thickness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>normal diabetes pedigree function</td>\n",
       "      <td>high age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>low pregnancies</td>\n",
       "      <td>normal glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>normal skin thickness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>normal diabetes pedigree function</td>\n",
       "      <td>normal age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>high pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>normal bmi</td>\n",
       "      <td>normal diabetes pedigree function</td>\n",
       "      <td>normal age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>low pregnancies</td>\n",
       "      <td>normal glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>normal skin thickness</td>\n",
       "      <td>normal insulin</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>low diabetes pedigree function</td>\n",
       "      <td>low age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>low pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>low blood pressure</td>\n",
       "      <td>high skin thickness</td>\n",
       "      <td>high insulin</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>high diabetes pedigree function</td>\n",
       "      <td>normal age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>763</td>\n",
       "      <td>0</td>\n",
       "      <td>high pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>high skin thickness</td>\n",
       "      <td>high insulin</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>low diabetes pedigree function</td>\n",
       "      <td>high age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>764</td>\n",
       "      <td>0</td>\n",
       "      <td>low pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>normal skin thickness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>normal diabetes pedigree function</td>\n",
       "      <td>low age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>765</td>\n",
       "      <td>0</td>\n",
       "      <td>normal pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>normal skin thickness</td>\n",
       "      <td>normal insulin</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>low diabetes pedigree function</td>\n",
       "      <td>normal age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>766</td>\n",
       "      <td>1</td>\n",
       "      <td>low pregnancies</td>\n",
       "      <td>high glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>normal diabetes pedigree function</td>\n",
       "      <td>high age</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>767</td>\n",
       "      <td>0</td>\n",
       "      <td>low pregnancies</td>\n",
       "      <td>normal glucose</td>\n",
       "      <td>normal blood pressure</td>\n",
       "      <td>high skin thickness</td>\n",
       "      <td>NaN</td>\n",
       "      <td>high bmi</td>\n",
       "      <td>normal diabetes pedigree function</td>\n",
       "      <td>low age</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>768 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     SampleNumber  Outcome  Pregnancies_Bucket  Glucose_Bucket  \\\n",
       "0               0        1  normal pregnancies    high glucose   \n",
       "1               1        0     low pregnancies  normal glucose   \n",
       "2               2        1    high pregnancies    high glucose   \n",
       "3               3        0     low pregnancies  normal glucose   \n",
       "4               4        1     low pregnancies    high glucose   \n",
       "..            ...      ...                 ...             ...   \n",
       "763           763        0    high pregnancies    high glucose   \n",
       "764           764        0     low pregnancies    high glucose   \n",
       "765           765        0  normal pregnancies    high glucose   \n",
       "766           766        1     low pregnancies    high glucose   \n",
       "767           767        0     low pregnancies  normal glucose   \n",
       "\n",
       "      BloodPressure_Bucket   SkinThickness_Bucket  Insulin_Bucket  BMI_Bucket  \\\n",
       "0    normal blood pressure    high skin thickness             NaN    high bmi   \n",
       "1    normal blood pressure  normal skin thickness             NaN    high bmi   \n",
       "2    normal blood pressure                    NaN             NaN  normal bmi   \n",
       "3    normal blood pressure  normal skin thickness  normal insulin    high bmi   \n",
       "4       low blood pressure    high skin thickness    high insulin    high bmi   \n",
       "..                     ...                    ...             ...         ...   \n",
       "763  normal blood pressure    high skin thickness    high insulin    high bmi   \n",
       "764  normal blood pressure  normal skin thickness             NaN    high bmi   \n",
       "765  normal blood pressure  normal skin thickness  normal insulin    high bmi   \n",
       "766  normal blood pressure                    NaN             NaN    high bmi   \n",
       "767  normal blood pressure    high skin thickness             NaN    high bmi   \n",
       "\n",
       "       DiabetesPedigreeFunction_Bucket  Age_Bucket  \n",
       "0    normal diabetes pedigree function    high age  \n",
       "1    normal diabetes pedigree function  normal age  \n",
       "2    normal diabetes pedigree function  normal age  \n",
       "3       low diabetes pedigree function     low age  \n",
       "4      high diabetes pedigree function  normal age  \n",
       "..                                 ...         ...  \n",
       "763     low diabetes pedigree function    high age  \n",
       "764  normal diabetes pedigree function     low age  \n",
       "765     low diabetes pedigree function  normal age  \n",
       "766  normal diabetes pedigree function    high age  \n",
       "767  normal diabetes pedigree function     low age  \n",
       "\n",
       "[768 rows x 10 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view df\n",
    "diabetes_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#one hot encode buckets and split embeddings\n",
    "combined_df[\"sampleNumber\"] = combined_df[\"sampleNumber\"].astype(int)\n",
    "combined_df = combined_df.merge(diabetes_df, left_on=\"sampleNumber\", right_on=\"SampleNumber\", how=\"left\")\n",
    "combined_df = combined_df.drop(columns=[\"sampleNumber\"])\n",
    "categorical_columns = ['Pregnancies_Bucket', 'Glucose_Bucket', 'BloodPressure_Bucket', 'SkinThickness_Bucket', 'Insulin_Bucket', 'BMI_Bucket', 'DiabetesPedigreeFunction_Bucket', 'Age_Bucket']\n",
    "processed_df = pd.get_dummies(combined_df, columns=categorical_columns, dtype=\"int\")\n",
    "embedding_df = pd.DataFrame(processed_df[\"embedding\"].tolist())\n",
    "embedding_df.columns = [f\"embedding_{i}\" for i in range(128)]\n",
    "final_df = pd.concat([processed_df, embedding_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nodeId</th>\n",
       "      <th>embedding</th>\n",
       "      <th>SampleNumber</th>\n",
       "      <th>Outcome</th>\n",
       "      <th>Pregnancies_Bucket_high pregnancies</th>\n",
       "      <th>Pregnancies_Bucket_low pregnancies</th>\n",
       "      <th>Pregnancies_Bucket_normal pregnancies</th>\n",
       "      <th>Glucose_Bucket_high glucose</th>\n",
       "      <th>Glucose_Bucket_low glucose</th>\n",
       "      <th>Glucose_Bucket_normal glucose</th>\n",
       "      <th>...</th>\n",
       "      <th>embedding_118</th>\n",
       "      <th>embedding_119</th>\n",
       "      <th>embedding_120</th>\n",
       "      <th>embedding_121</th>\n",
       "      <th>embedding_122</th>\n",
       "      <th>embedding_123</th>\n",
       "      <th>embedding_124</th>\n",
       "      <th>embedding_125</th>\n",
       "      <th>embedding_126</th>\n",
       "      <th>embedding_127</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[0.4141680598258972, 0.03869418054819107, -0.1...</td>\n",
       "      <td>382</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.083448</td>\n",
       "      <td>0.172059</td>\n",
       "      <td>0.088233</td>\n",
       "      <td>0.206619</td>\n",
       "      <td>-0.007406</td>\n",
       "      <td>0.212477</td>\n",
       "      <td>0.029658</td>\n",
       "      <td>-0.024647</td>\n",
       "      <td>0.052844</td>\n",
       "      <td>-0.103768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[0.371269166469574, 0.05201292037963867, -0.10...</td>\n",
       "      <td>383</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.123794</td>\n",
       "      <td>0.151029</td>\n",
       "      <td>0.061160</td>\n",
       "      <td>0.154785</td>\n",
       "      <td>0.001946</td>\n",
       "      <td>0.171825</td>\n",
       "      <td>0.103335</td>\n",
       "      <td>0.017990</td>\n",
       "      <td>-0.007571</td>\n",
       "      <td>-0.114803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[0.3478549122810364, 0.06399303674697876, -0.1...</td>\n",
       "      <td>384</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.088545</td>\n",
       "      <td>0.199655</td>\n",
       "      <td>0.058721</td>\n",
       "      <td>0.136495</td>\n",
       "      <td>-0.044946</td>\n",
       "      <td>0.187021</td>\n",
       "      <td>0.042974</td>\n",
       "      <td>0.005521</td>\n",
       "      <td>0.059122</td>\n",
       "      <td>-0.160641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[0.4099091589450836, 0.027811255306005478, -0....</td>\n",
       "      <td>385</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.088704</td>\n",
       "      <td>0.242223</td>\n",
       "      <td>0.023763</td>\n",
       "      <td>0.111797</td>\n",
       "      <td>0.030774</td>\n",
       "      <td>0.150176</td>\n",
       "      <td>0.063015</td>\n",
       "      <td>-0.000636</td>\n",
       "      <td>0.053633</td>\n",
       "      <td>-0.177662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[0.3573867976665497, 0.0758630633354187, 0.027...</td>\n",
       "      <td>386</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.066840</td>\n",
       "      <td>0.182776</td>\n",
       "      <td>0.129548</td>\n",
       "      <td>0.206703</td>\n",
       "      <td>-0.061997</td>\n",
       "      <td>0.242070</td>\n",
       "      <td>0.059771</td>\n",
       "      <td>-0.012242</td>\n",
       "      <td>-0.007600</td>\n",
       "      <td>-0.151231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>1152</td>\n",
       "      <td>[0.4162055253982544, 0.026111219078302383, -0....</td>\n",
       "      <td>377</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.062308</td>\n",
       "      <td>0.200209</td>\n",
       "      <td>0.078861</td>\n",
       "      <td>0.161985</td>\n",
       "      <td>-0.032400</td>\n",
       "      <td>0.201345</td>\n",
       "      <td>0.040949</td>\n",
       "      <td>0.010791</td>\n",
       "      <td>-0.033868</td>\n",
       "      <td>-0.073606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>1153</td>\n",
       "      <td>[0.34819433093070984, 0.0923713967204094, 0.03...</td>\n",
       "      <td>378</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.049506</td>\n",
       "      <td>0.229343</td>\n",
       "      <td>0.161155</td>\n",
       "      <td>0.272059</td>\n",
       "      <td>-0.061438</td>\n",
       "      <td>0.229603</td>\n",
       "      <td>0.063524</td>\n",
       "      <td>-0.005243</td>\n",
       "      <td>0.041468</td>\n",
       "      <td>-0.142256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>1156</td>\n",
       "      <td>[0.4478495121002197, 0.06772170960903168, -0.0...</td>\n",
       "      <td>379</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.117071</td>\n",
       "      <td>0.217234</td>\n",
       "      <td>0.099096</td>\n",
       "      <td>0.223523</td>\n",
       "      <td>0.048493</td>\n",
       "      <td>0.165011</td>\n",
       "      <td>0.162497</td>\n",
       "      <td>-0.015010</td>\n",
       "      <td>-0.026475</td>\n",
       "      <td>-0.101034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>1157</td>\n",
       "      <td>[0.3707543611526489, 0.054082226008176804, -0....</td>\n",
       "      <td>380</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.091443</td>\n",
       "      <td>0.208174</td>\n",
       "      <td>0.108655</td>\n",
       "      <td>0.221055</td>\n",
       "      <td>0.000519</td>\n",
       "      <td>0.197458</td>\n",
       "      <td>0.047120</td>\n",
       "      <td>-0.000868</td>\n",
       "      <td>0.022656</td>\n",
       "      <td>-0.060568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>1159</td>\n",
       "      <td>[0.3539871275424957, 0.05613316223025322, -0.1...</td>\n",
       "      <td>381</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.097113</td>\n",
       "      <td>0.180614</td>\n",
       "      <td>0.057672</td>\n",
       "      <td>0.142338</td>\n",
       "      <td>-0.042510</td>\n",
       "      <td>0.172537</td>\n",
       "      <td>0.047556</td>\n",
       "      <td>0.004736</td>\n",
       "      <td>0.078431</td>\n",
       "      <td>-0.174583</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>768 rows × 156 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     nodeId                                          embedding  SampleNumber  \\\n",
       "0         0  [0.4141680598258972, 0.03869418054819107, -0.1...           382   \n",
       "1         1  [0.371269166469574, 0.05201292037963867, -0.10...           383   \n",
       "2         2  [0.3478549122810364, 0.06399303674697876, -0.1...           384   \n",
       "3         3  [0.4099091589450836, 0.027811255306005478, -0....           385   \n",
       "4         4  [0.3573867976665497, 0.0758630633354187, 0.027...           386   \n",
       "..      ...                                                ...           ...   \n",
       "763    1152  [0.4162055253982544, 0.026111219078302383, -0....           377   \n",
       "764    1153  [0.34819433093070984, 0.0923713967204094, 0.03...           378   \n",
       "765    1156  [0.4478495121002197, 0.06772170960903168, -0.0...           379   \n",
       "766    1157  [0.3707543611526489, 0.054082226008176804, -0....           380   \n",
       "767    1159  [0.3539871275424957, 0.05613316223025322, -0.1...           381   \n",
       "\n",
       "     Outcome  Pregnancies_Bucket_high pregnancies  \\\n",
       "0          0                                    0   \n",
       "1          0                                    0   \n",
       "2          0                                    0   \n",
       "3          0                                    0   \n",
       "4          1                                    0   \n",
       "..       ...                                  ...   \n",
       "763        0                                    0   \n",
       "764        1                                    0   \n",
       "765        0                                    0   \n",
       "766        0                                    0   \n",
       "767        0                                    0   \n",
       "\n",
       "     Pregnancies_Bucket_low pregnancies  \\\n",
       "0                                     1   \n",
       "1                                     1   \n",
       "2                                     1   \n",
       "3                                     1   \n",
       "4                                     0   \n",
       "..                                  ...   \n",
       "763                                   1   \n",
       "764                                   0   \n",
       "765                                   1   \n",
       "766                                   1   \n",
       "767                                   1   \n",
       "\n",
       "     Pregnancies_Bucket_normal pregnancies  Glucose_Bucket_high glucose  \\\n",
       "0                                        0                            1   \n",
       "1                                        0                            0   \n",
       "2                                        0                            1   \n",
       "3                                        0                            1   \n",
       "4                                        1                            1   \n",
       "..                                     ...                          ...   \n",
       "763                                      0                            0   \n",
       "764                                      1                            1   \n",
       "765                                      0                            0   \n",
       "766                                      0                            1   \n",
       "767                                      0                            1   \n",
       "\n",
       "     Glucose_Bucket_low glucose  Glucose_Bucket_normal glucose  ...  \\\n",
       "0                             0                              0  ...   \n",
       "1                             0                              1  ...   \n",
       "2                             0                              0  ...   \n",
       "3                             0                              0  ...   \n",
       "4                             0                              0  ...   \n",
       "..                          ...                            ...  ...   \n",
       "763                           0                              1  ...   \n",
       "764                           0                              0  ...   \n",
       "765                           0                              1  ...   \n",
       "766                           0                              0  ...   \n",
       "767                           0                              0  ...   \n",
       "\n",
       "     embedding_118  embedding_119  embedding_120  embedding_121  \\\n",
       "0        -0.083448       0.172059       0.088233       0.206619   \n",
       "1        -0.123794       0.151029       0.061160       0.154785   \n",
       "2        -0.088545       0.199655       0.058721       0.136495   \n",
       "3        -0.088704       0.242223       0.023763       0.111797   \n",
       "4        -0.066840       0.182776       0.129548       0.206703   \n",
       "..             ...            ...            ...            ...   \n",
       "763      -0.062308       0.200209       0.078861       0.161985   \n",
       "764      -0.049506       0.229343       0.161155       0.272059   \n",
       "765      -0.117071       0.217234       0.099096       0.223523   \n",
       "766      -0.091443       0.208174       0.108655       0.221055   \n",
       "767      -0.097113       0.180614       0.057672       0.142338   \n",
       "\n",
       "     embedding_122  embedding_123  embedding_124  embedding_125  \\\n",
       "0        -0.007406       0.212477       0.029658      -0.024647   \n",
       "1         0.001946       0.171825       0.103335       0.017990   \n",
       "2        -0.044946       0.187021       0.042974       0.005521   \n",
       "3         0.030774       0.150176       0.063015      -0.000636   \n",
       "4        -0.061997       0.242070       0.059771      -0.012242   \n",
       "..             ...            ...            ...            ...   \n",
       "763      -0.032400       0.201345       0.040949       0.010791   \n",
       "764      -0.061438       0.229603       0.063524      -0.005243   \n",
       "765       0.048493       0.165011       0.162497      -0.015010   \n",
       "766       0.000519       0.197458       0.047120      -0.000868   \n",
       "767      -0.042510       0.172537       0.047556       0.004736   \n",
       "\n",
       "     embedding_126  embedding_127  \n",
       "0         0.052844      -0.103768  \n",
       "1        -0.007571      -0.114803  \n",
       "2         0.059122      -0.160641  \n",
       "3         0.053633      -0.177662  \n",
       "4        -0.007600      -0.151231  \n",
       "..             ...            ...  \n",
       "763      -0.033868      -0.073606  \n",
       "764       0.041468      -0.142256  \n",
       "765      -0.026475      -0.101034  \n",
       "766       0.022656      -0.060568  \n",
       "767       0.078431      -0.174583  \n",
       "\n",
       "[768 rows x 156 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#view df\n",
    "final_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#optional - load final df if you want to do custom training on our embeddings (ie. you can skip the above sections and train on our embeddings if you want to skip the neo4j steps)\n",
    "#final_df = pd.read_csv(\"data/final_df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split dataset into x and y (y being outcome)\n",
    "X = final_df.loc[:,~final_df.columns.isin([\"Outcome\"])]\n",
    "y = final_df[\"Outcome\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV,  StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup parameter grid for gridsearch\n",
    "param_grid = {\n",
    "    'n_estimators': [100, 200, 300], \n",
    "    'max_depth': [5, 10, 15],       \n",
    "    'min_samples_split': [2, 5, 10], \n",
    "    'min_samples_leaf': [1, 2, 4]    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup rf model (using stratified cross validation and gridsearch)\n",
    "rf = RandomForestClassifier(random_state=42)\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#specify categorical columns\n",
    "categorical_columns = [ 'Pregnancies_Bucket_high pregnancies',\n",
    "       'Pregnancies_Bucket_low pregnancies',\n",
    "       'Pregnancies_Bucket_normal pregnancies', 'Glucose_Bucket_high glucose',\n",
    "       'Glucose_Bucket_low glucose', 'Glucose_Bucket_normal glucose',\n",
    "       'BloodPressure_Bucket_high blood pressure',\n",
    "       'BloodPressure_Bucket_low blood pressure',\n",
    "       'BloodPressure_Bucket_normal blood pressure',\n",
    "       'SkinThickness_Bucket_high skin thickness',\n",
    "       'SkinThickness_Bucket_low skin thickness',\n",
    "       'SkinThickness_Bucket_normal skin thickness',\n",
    "       'Insulin_Bucket_high insulin', 'Insulin_Bucket_low insulin',\n",
    "       'Insulin_Bucket_normal insulin', 'BMI_Bucket_high bmi',\n",
    "       'BMI_Bucket_low bmi', 'BMI_Bucket_normal bmi',\n",
    "       'DiabetesPedigreeFunction_Bucket_high diabetes pedigree function',\n",
    "       'DiabetesPedigreeFunction_Bucket_low diabetes pedigree function',\n",
    "       'DiabetesPedigreeFunction_Bucket_normal diabetes pedigree function',\n",
    "       'Age_Bucket_high age', 'Age_Bucket_low age', 'Age_Bucket_normal age']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies_Bucket_high pregnancies</th>\n",
       "      <th>Pregnancies_Bucket_low pregnancies</th>\n",
       "      <th>Pregnancies_Bucket_normal pregnancies</th>\n",
       "      <th>Glucose_Bucket_high glucose</th>\n",
       "      <th>Glucose_Bucket_low glucose</th>\n",
       "      <th>Glucose_Bucket_normal glucose</th>\n",
       "      <th>BloodPressure_Bucket_high blood pressure</th>\n",
       "      <th>BloodPressure_Bucket_low blood pressure</th>\n",
       "      <th>BloodPressure_Bucket_normal blood pressure</th>\n",
       "      <th>SkinThickness_Bucket_high skin thickness</th>\n",
       "      <th>...</th>\n",
       "      <th>Insulin_Bucket_normal insulin</th>\n",
       "      <th>BMI_Bucket_high bmi</th>\n",
       "      <th>BMI_Bucket_low bmi</th>\n",
       "      <th>BMI_Bucket_normal bmi</th>\n",
       "      <th>DiabetesPedigreeFunction_Bucket_high diabetes pedigree function</th>\n",
       "      <th>DiabetesPedigreeFunction_Bucket_low diabetes pedigree function</th>\n",
       "      <th>DiabetesPedigreeFunction_Bucket_normal diabetes pedigree function</th>\n",
       "      <th>Age_Bucket_high age</th>\n",
       "      <th>Age_Bucket_low age</th>\n",
       "      <th>Age_Bucket_normal age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>764</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>765</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>768 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pregnancies_Bucket_high pregnancies  Pregnancies_Bucket_low pregnancies  \\\n",
       "0                                      0                                   1   \n",
       "1                                      0                                   1   \n",
       "2                                      0                                   1   \n",
       "3                                      0                                   1   \n",
       "4                                      0                                   0   \n",
       "..                                   ...                                 ...   \n",
       "763                                    0                                   1   \n",
       "764                                    0                                   0   \n",
       "765                                    0                                   1   \n",
       "766                                    0                                   1   \n",
       "767                                    0                                   1   \n",
       "\n",
       "     Pregnancies_Bucket_normal pregnancies  Glucose_Bucket_high glucose  \\\n",
       "0                                        0                            1   \n",
       "1                                        0                            0   \n",
       "2                                        0                            1   \n",
       "3                                        0                            1   \n",
       "4                                        1                            1   \n",
       "..                                     ...                          ...   \n",
       "763                                      0                            0   \n",
       "764                                      1                            1   \n",
       "765                                      0                            0   \n",
       "766                                      0                            1   \n",
       "767                                      0                            1   \n",
       "\n",
       "     Glucose_Bucket_low glucose  Glucose_Bucket_normal glucose  \\\n",
       "0                             0                              0   \n",
       "1                             0                              1   \n",
       "2                             0                              0   \n",
       "3                             0                              0   \n",
       "4                             0                              0   \n",
       "..                          ...                            ...   \n",
       "763                           0                              1   \n",
       "764                           0                              0   \n",
       "765                           0                              1   \n",
       "766                           0                              0   \n",
       "767                           0                              0   \n",
       "\n",
       "     BloodPressure_Bucket_high blood pressure  \\\n",
       "0                                           0   \n",
       "1                                           0   \n",
       "2                                           0   \n",
       "3                                           0   \n",
       "4                                           0   \n",
       "..                                        ...   \n",
       "763                                         0   \n",
       "764                                         0   \n",
       "765                                         1   \n",
       "766                                         0   \n",
       "767                                         0   \n",
       "\n",
       "     BloodPressure_Bucket_low blood pressure  \\\n",
       "0                                          0   \n",
       "1                                          0   \n",
       "2                                          0   \n",
       "3                                          1   \n",
       "4                                          0   \n",
       "..                                       ...   \n",
       "763                                        0   \n",
       "764                                        0   \n",
       "765                                        0   \n",
       "766                                        0   \n",
       "767                                        0   \n",
       "\n",
       "     BloodPressure_Bucket_normal blood pressure  \\\n",
       "0                                             1   \n",
       "1                                             1   \n",
       "2                                             1   \n",
       "3                                             0   \n",
       "4                                             1   \n",
       "..                                          ...   \n",
       "763                                           1   \n",
       "764                                           1   \n",
       "765                                           0   \n",
       "766                                           1   \n",
       "767                                           1   \n",
       "\n",
       "     SkinThickness_Bucket_high skin thickness  ...  \\\n",
       "0                                           0  ...   \n",
       "1                                           0  ...   \n",
       "2                                           0  ...   \n",
       "3                                           0  ...   \n",
       "4                                           0  ...   \n",
       "..                                        ...  ...   \n",
       "763                                         1  ...   \n",
       "764                                         0  ...   \n",
       "765                                         1  ...   \n",
       "766                                         1  ...   \n",
       "767                                         0  ...   \n",
       "\n",
       "     Insulin_Bucket_normal insulin  BMI_Bucket_high bmi  BMI_Bucket_low bmi  \\\n",
       "0                                0                    1                   0   \n",
       "1                                1                    1                   0   \n",
       "2                                1                    0                   0   \n",
       "3                                1                    0                   0   \n",
       "4                                0                    1                   0   \n",
       "..                             ...                  ...                 ...   \n",
       "763                              1                    1                   0   \n",
       "764                              0                    1                   0   \n",
       "765                              1                    1                   0   \n",
       "766                              1                    1                   0   \n",
       "767                              0                    0                   0   \n",
       "\n",
       "     BMI_Bucket_normal bmi  \\\n",
       "0                        0   \n",
       "1                        0   \n",
       "2                        1   \n",
       "3                        1   \n",
       "4                        0   \n",
       "..                     ...   \n",
       "763                      0   \n",
       "764                      0   \n",
       "765                      0   \n",
       "766                      0   \n",
       "767                      1   \n",
       "\n",
       "     DiabetesPedigreeFunction_Bucket_high diabetes pedigree function  \\\n",
       "0                                                    1                 \n",
       "1                                                    1                 \n",
       "2                                                    0                 \n",
       "3                                                    0                 \n",
       "4                                                    0                 \n",
       "..                                                 ...                 \n",
       "763                                                  0                 \n",
       "764                                                  0                 \n",
       "765                                                  1                 \n",
       "766                                                  1                 \n",
       "767                                                  0                 \n",
       "\n",
       "     DiabetesPedigreeFunction_Bucket_low diabetes pedigree function  \\\n",
       "0                                                    0                \n",
       "1                                                    0                \n",
       "2                                                    1                \n",
       "3                                                    1                \n",
       "4                                                    0                \n",
       "..                                                 ...                \n",
       "763                                                  0                \n",
       "764                                                  1                \n",
       "765                                                  0                \n",
       "766                                                  0                \n",
       "767                                                  1                \n",
       "\n",
       "     DiabetesPedigreeFunction_Bucket_normal diabetes pedigree function  \\\n",
       "0                                                    0                   \n",
       "1                                                    0                   \n",
       "2                                                    0                   \n",
       "3                                                    0                   \n",
       "4                                                    1                   \n",
       "..                                                 ...                   \n",
       "763                                                  1                   \n",
       "764                                                  0                   \n",
       "765                                                  0                   \n",
       "766                                                  0                   \n",
       "767                                                  0                   \n",
       "\n",
       "     Age_Bucket_high age  Age_Bucket_low age  Age_Bucket_normal age  \n",
       "0                      0                   1                      0  \n",
       "1                      0                   1                      0  \n",
       "2                      0                   1                      0  \n",
       "3                      0                   1                      0  \n",
       "4                      0                   0                      1  \n",
       "..                   ...                 ...                    ...  \n",
       "763                    0                   1                      0  \n",
       "764                    0                   0                      1  \n",
       "765                    0                   0                      1  \n",
       "766                    0                   1                      0  \n",
       "767                    0                   1                      0  \n",
       "\n",
       "[768 rows x 24 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#select categorical columns only for bucketed model\n",
    "X[categorical_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7804    0.8600    0.8183       500\n",
      "           1     0.6774    0.5485    0.6062       268\n",
      "\n",
      "    accuracy                         0.7513       768\n",
      "   macro avg     0.7289    0.7043    0.7122       768\n",
      "weighted avg     0.7445    0.7513    0.7443       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply bucketed model and print out classification report\n",
    "grid_search = GridSearchCV(rf, param_grid, scoring='accuracy', cv=cv) \n",
    "grid_search.fit(X[categorical_columns], y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[categorical_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/rfbucket.pkl']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/rfbucket.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#specify embedding columns\n",
    "embedding_columns = [f'embedding_{i}' for i in range(embedding_dimension)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8702    0.9120    0.8906       500\n",
      "           1     0.8197    0.7463    0.7812       268\n",
      "\n",
      "    accuracy                         0.8542       768\n",
      "   macro avg     0.8450    0.8291    0.8359       768\n",
      "weighted avg     0.8526    0.8542    0.8525       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply embedding model and print out classification report\n",
    "grid_search = GridSearchCV(rf, param_grid, scoring='accuracy', cv=cv) \n",
    "grid_search.fit(X[embedding_columns], y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[embedding_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/rfemb.pkl']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/rfemb.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "#setup parameter grid for gridsearch\n",
    "param_grid = {\n",
    "  'n_estimators': [100, 200, 300], \n",
    "  'max_depth': [5, 10, 15],       \n",
    "  'min_child_weight': [2, 5, 10], \n",
    "  'gamma': [0, 0.1, 0.5]          \n",
    "}\n",
    "\n",
    "xgb_model = XGBClassifier(objective='binary:logistic', random_state=42)\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7876    0.8380    0.8120       500\n",
      "           1     0.6568    0.5784    0.6151       268\n",
      "\n",
      "    accuracy                         0.7474       768\n",
      "   macro avg     0.7222    0.7082    0.7135       768\n",
      "weighted avg     0.7419    0.7474    0.7433       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply bucketed model and print out classification report\n",
    "grid_search = GridSearchCV(xgb_model, param_grid, scoring='accuracy', cv=cv)\n",
    "grid_search.fit(X[categorical_columns], y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[categorical_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/xgbbucket.pkl']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/xgbbucket.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8835    0.9100    0.8966       500\n",
      "           1     0.8221    0.7761    0.7985       268\n",
      "\n",
      "    accuracy                         0.8633       768\n",
      "   macro avg     0.8528    0.8431    0.8475       768\n",
      "weighted avg     0.8621    0.8633    0.8623       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply embedding model and print out classification report\n",
    "grid_search = GridSearchCV(xgb_model, param_grid, scoring='accuracy', cv=cv)\n",
    "grid_search.fit(X[embedding_columns], y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[embedding_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/xgbemb.pkl']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/xgbemb.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "#setup parameter grid for gridsearch\n",
    "param_grid = {\n",
    "    'C': [0.1, 1, 10],\n",
    "    'kernel': ['linear', 'rbf'],\n",
    "    'gamma': [0.001, 0.01, 0.1]\n",
    "}\n",
    "\n",
    "svm = SVC(random_state=42)\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7936    0.7920    0.7928       500\n",
      "           1     0.6134    0.6157    0.6145       268\n",
      "\n",
      "    accuracy                         0.7305       768\n",
      "   macro avg     0.7035    0.7038    0.7037       768\n",
      "weighted avg     0.7307    0.7305    0.7306       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply bucketed model and print out classification report\n",
    "grid_search = GridSearchCV(svm, param_grid, scoring='accuracy', cv=cv) \n",
    "grid_search.fit(X[categorical_columns], y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[categorical_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/svmbucket.pkl']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/svmbucket.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7984    0.7840    0.7911       500\n",
      "           1     0.6101    0.6306    0.6202       268\n",
      "\n",
      "    accuracy                         0.7305       768\n",
      "   macro avg     0.7042    0.7073    0.7057       768\n",
      "weighted avg     0.7327    0.7305    0.7315       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply embedding model and print out classification report\n",
    "grid_search = GridSearchCV(svm, param_grid, scoring='accuracy', cv=cv) \n",
    "grid_search.fit(X[embedding_columns], y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[embedding_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/svmemb.pkl']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/svmemb.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB \n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "\n",
    "#setup parameter grid for gridsearch\n",
    "param_grid = {\n",
    "  'var_smoothing': [1e-9, 1e-8, 1e-7, 1e-6] \n",
    "}\n",
    "\n",
    "nb = GaussianNB()\n",
    "cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.9179    0.5140    0.6590       500\n",
      "           1     0.5020    0.9142    0.6481       268\n",
      "\n",
      "    accuracy                         0.6536       768\n",
      "   macro avg     0.7100    0.7141    0.6536       768\n",
      "weighted avg     0.7728    0.6536    0.6552       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply bucketed model and print out classification report\n",
    "grid_search = GridSearchCV(nb, param_grid, cv=cv, scoring='accuracy')\n",
    "grid_search.fit(X[categorical_columns],y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[categorical_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/nbbucket.pkl']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/nbbucket.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.8501    0.6580    0.7418       500\n",
      "           1     0.5512    0.7836    0.6471       268\n",
      "\n",
      "    accuracy                         0.7018       768\n",
      "   macro avg     0.7007    0.7208    0.6945       768\n",
      "weighted avg     0.7458    0.7018    0.7088       768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#apply embedding model and print out classification report\n",
    "grid_search = GridSearchCV(nb, param_grid, cv=cv, scoring='accuracy')\n",
    "grid_search.fit(X[embedding_columns],y)\n",
    "\n",
    "best_model = grid_search.best_estimator_\n",
    "best_params = grid_search.best_params_\n",
    "\n",
    "y_pred = best_model.predict(X[embedding_columns])\n",
    "print(classification_report(y, y_pred, digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_models/nbemb.pkl']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save best model for later use\n",
    "joblib.dump(best_model, 'final_models/nbemb.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "from sklearn.model_selection import KFold\n",
    "import numpy as np\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout, BatchNormalization\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup nn variables\n",
    "tf.random.set_seed(42)\n",
    "batch_size = 8\n",
    "epochs = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model defintion\n",
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(64, activation=\"sigmoid\"))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Dense(128, activation=\"sigmoid\"))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Dense(64, activation=\"sigmoid\"))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Dense(1, activation=\"sigmoid\"))\n",
    "    model.compile(loss=\"binary_crossentropy\", metrics=[\"accuracy\"], optimizer=\"adam\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#split dataset using a 80/10/10 split\n",
    "X_train, X_val_test, y_train, y_val_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n",
    "X_val, X_test, y_val, y_test = train_test_split(X_val_test, y_val_test, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.6080 - loss: 0.7821 - val_accuracy: 0.6494 - val_loss: 0.6513 - learning_rate: 0.0010\n",
      "Epoch 2/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.6712 - loss: 0.6448 - val_accuracy: 0.6494 - val_loss: 0.6320 - learning_rate: 0.0010\n",
      "Epoch 3/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7058 - loss: 0.5708 - val_accuracy: 0.6494 - val_loss: 0.6120 - learning_rate: 0.0010\n",
      "Epoch 4/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.6940 - loss: 0.6001 - val_accuracy: 0.6234 - val_loss: 0.5928 - learning_rate: 0.0010\n",
      "Epoch 5/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.6968 - loss: 0.5707 - val_accuracy: 0.6364 - val_loss: 0.5885 - learning_rate: 0.0010\n",
      "Epoch 6/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7324 - loss: 0.5714 - val_accuracy: 0.6364 - val_loss: 0.6155 - learning_rate: 0.0010\n",
      "Epoch 7/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7193 - loss: 0.5310 - val_accuracy: 0.6753 - val_loss: 0.6456 - learning_rate: 0.0010\n",
      "Epoch 8/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7099 - loss: 0.5389 - val_accuracy: 0.6883 - val_loss: 0.6302 - learning_rate: 0.0010\n",
      "Epoch 9/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7698 - loss: 0.4752 - val_accuracy: 0.6753 - val_loss: 0.6313 - learning_rate: 5.0000e-04\n",
      "Epoch 10/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7424 - loss: 0.5181 - val_accuracy: 0.6623 - val_loss: 0.6182 - learning_rate: 5.0000e-04\n",
      "\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7100 - loss: 0.5403 \n",
      "Test Accuracy: 0.701298713684082\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\n",
    "\n",
    "\n",
    "# Convert data to numpy arrays\n",
    "X_train_data = np.array(X_train[categorical_columns])\n",
    "y_train_data = np.array(y_train)\n",
    "X_val_data = np.array(X_val[categorical_columns])\n",
    "y_val_data = np.array(y_val)\n",
    "X_test_data = np.array(X_test[categorical_columns])\n",
    "y_test_data = np.array(y_test)\n",
    "\n",
    "checkpoint_file = 'final_models/nnbucket.keras'\n",
    "\n",
    "\n",
    "# Create a new instance of the model\n",
    "model = create_model()\n",
    "\n",
    "# Define callbacks\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=0.0001)\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n",
    "model_checkpoint = ModelCheckpoint(checkpoint_file, monitor='val_loss', save_best_only=True, mode='min')\n",
    "\n",
    "# Train the model\n",
    "hist = model.fit(\n",
    "    X_train_data, y_train_data,\n",
    "    batch_size=batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=(X_val_data, y_val_data),\n",
    "    callbacks=[reduce_lr, early_stopping, model_checkpoint]\n",
    ")\n",
    "\n",
    "# Evaluate the model on the test set\n",
    "test_prediction_accuracy = model.evaluate(X_test_data, y_test_data)[1]\n",
    "print(f\"Test Accuracy: {test_prediction_accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 36ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7288    0.8600    0.7890        50\n",
      "           1     0.6111    0.4074    0.4889        27\n",
      "\n",
      "    accuracy                         0.7013        77\n",
      "   macro avg     0.6700    0.6337    0.6389        77\n",
      "weighted avg     0.6875    0.7013    0.6838        77\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "#load best model to get classification report\n",
    "X_test_data = np.array(X_test[categorical_columns])\n",
    "y_test_data = np.array(y_test)\n",
    "model = load_model('final_models/nnbucket.keras')\n",
    "\n",
    "y_pred = model.predict(X_test_data)  \n",
    "y_pred_2 = (y_pred>.5).astype(int) \n",
    "\n",
    "report = classification_report(y_test_data, y_pred_2, digits=4)\n",
    "print(report)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m3s\u001b[0m 4ms/step - accuracy: 0.6386 - loss: 0.7480 - val_accuracy: 0.3506 - val_loss: 0.7385 - learning_rate: 0.0010\n",
      "Epoch 2/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.6542 - loss: 0.6616 - val_accuracy: 0.6494 - val_loss: 0.6789 - learning_rate: 0.0010\n",
      "Epoch 3/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.6989 - loss: 0.5632 - val_accuracy: 0.6494 - val_loss: 0.6480 - learning_rate: 0.0010\n",
      "Epoch 4/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7045 - loss: 0.5740 - val_accuracy: 0.6494 - val_loss: 0.6270 - learning_rate: 0.0010\n",
      "Epoch 5/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7099 - loss: 0.5584 - val_accuracy: 0.6494 - val_loss: 0.6018 - learning_rate: 0.0010\n",
      "Epoch 6/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7307 - loss: 0.5487 - val_accuracy: 0.6364 - val_loss: 0.5795 - learning_rate: 0.0010\n",
      "Epoch 7/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7266 - loss: 0.5317 - val_accuracy: 0.6364 - val_loss: 0.5776 - learning_rate: 0.0010\n",
      "Epoch 8/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7292 - loss: 0.5342 - val_accuracy: 0.6494 - val_loss: 0.6001 - learning_rate: 0.0010\n",
      "Epoch 9/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7249 - loss: 0.5366 - val_accuracy: 0.6883 - val_loss: 0.6248 - learning_rate: 0.0010\n",
      "Epoch 10/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7146 - loss: 0.5374 - val_accuracy: 0.6494 - val_loss: 0.6388 - learning_rate: 0.0010\n",
      "Epoch 11/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7107 - loss: 0.5339 - val_accuracy: 0.6364 - val_loss: 0.6254 - learning_rate: 5.0000e-04\n",
      "Epoch 12/30\n",
      "\u001b[1m77/77\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7513 - loss: 0.5240 - val_accuracy: 0.6623 - val_loss: 0.6356 - learning_rate: 5.0000e-04\n",
      "\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 2ms/step - accuracy: 0.7555 - loss: 0.5116 \n",
      "Test Accuracy: 0.7532467246055603\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\n",
    "\n",
    "# Convert data to numpy arrays\n",
    "X_train_data = np.array(X_train[embedding_columns])\n",
    "y_train_data = np.array(y_train)\n",
    "X_val_data = np.array(X_val[embedding_columns])\n",
    "y_val_data = np.array(y_val)\n",
    "X_test_data = np.array(X_test[embedding_columns])\n",
    "y_test_data = np.array(y_test)\n",
    "\n",
    "checkpoint_file = 'final_models/nnemb.keras'\n",
    "\n",
    "\n",
    "# Create a new instance of the model\n",
    "model = create_model()\n",
    "\n",
    "# Define callbacks\n",
    "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=0.0001)\n",
    "early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n",
    "model_checkpoint = ModelCheckpoint(checkpoint_file, monitor='val_loss', save_best_only=True, mode='min')\n",
    "\n",
    "# Train the model\n",
    "hist = model.fit(\n",
    "    X_train_data, y_train_data,\n",
    "    batch_size=batch_size,\n",
    "    epochs=epochs,\n",
    "    validation_data=(X_val_data, y_val_data),\n",
    "    callbacks=[reduce_lr, early_stopping, model_checkpoint]\n",
    ")\n",
    "\n",
    "# Evaluate the model on the test set\n",
    "test_prediction_accuracy = model.evaluate(X_test_data, y_test_data)[1]\n",
    "print(f\"Test Accuracy: {test_prediction_accuracy}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:5 out of the last 7 calls to <function TensorFlowTrainer.make_predict_function.<locals>.one_step_on_data_distributed at 0x000001C56B51EB60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "\u001b[1m1/3\u001b[0m \u001b[32m━━━━━━\u001b[0m\u001b[37m━━━━━━━━━━━━━━\u001b[0m \u001b[1m0s\u001b[0m 84ms/stepWARNING:tensorflow:6 out of the last 9 calls to <function TensorFlowTrainer.make_predict_function.<locals>.one_step_on_data_distributed at 0x000001C56B51EB60> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n",
      "\u001b[1m3/3\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m0s\u001b[0m 34ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.7719    0.8800    0.8224        50\n",
      "           1     0.7000    0.5185    0.5957        27\n",
      "\n",
      "    accuracy                         0.7532        77\n",
      "   macro avg     0.7360    0.6993    0.7091        77\n",
      "weighted avg     0.7467    0.7532    0.7429        77\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "#load best model to get classification report\n",
    "X_test_data = np.array(X_test[embedding_columns])\n",
    "y_test_data = np.array(y_test)\n",
    "model = load_model('final_models/nnemb.keras')\n",
    "\n",
    "y_pred = model.predict(X_test_data)  \n",
    "y_pred_2 = (y_pred>.5).astype(int) \n",
    "\n",
    "report = classification_report(y_test_data, y_pred_2, digits=4)\n",
    "print(report)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}