notebooks/knn_cosine_levenshtein_threadpool.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "311237c2",
   "metadata": {},
   "source": [
    "## Predict Race/Ethnicity of Unseen Last Name Using KNN (Cosine Distance and Levenshtein)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "96b61aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import Levenshtein as lv\n",
    "import multiprocessing as mp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer                                                             \n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "31452866",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>walker</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>palmer</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mc cleod</td>\n",
       "      <td>nh_black</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>scarborough</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>walker</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653889</th>\n",
       "      <td>walters</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653890</th>\n",
       "      <td>sawyer</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653891</th>\n",
       "      <td>thomas</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653892</th>\n",
       "      <td>campbell</td>\n",
       "      <td>multi_racial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13653893</th>\n",
       "      <td>bruner</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13336926 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            name_last          race\n",
       "0              walker      nh_white\n",
       "1              palmer      nh_white\n",
       "2            mc cleod      nh_black\n",
       "3         scarborough      nh_white\n",
       "4              walker      nh_white\n",
       "...               ...           ...\n",
       "13653889      walters      nh_white\n",
       "13653890       sawyer      nh_white\n",
       "13653891       thomas      nh_white\n",
       "13653892     campbell  multi_racial\n",
       "13653893       bruner      nh_white\n",
       "\n",
       "[13336926 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Florida voter file\n",
    "df = pd.read_csv('./dataverse_files/fl_reg_name_race.csv.gz', usecols=['name_last', 'race'])\n",
    "df.dropna(subset=['name_last'], inplace=True)\n",
    "\n",
    "# We assume unknown as missing at random\n",
    "sdf = df[df.race.isin(['unknown']) == False]\n",
    "del df\n",
    "\n",
    "# Setting consistent case for names\n",
    "sdf['name_last'] = sdf.name_last.str.strip().str.lower()\n",
    "\n",
    "sdf\n",
    "\n",
    "#sdf = df[df.race.isin(['multi_racial', 'native_indian', 'other', 'unknown']) == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d7d244de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nh_white         8714118\n",
       "hispanic         2174408\n",
       "nh_black         1847266\n",
       "asian             253306\n",
       "other             208250\n",
       "multi_racial       94119\n",
       "native_indian      45459\n",
       "Name: race, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the different races filtered\n",
    "sdf.race.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "787aac23",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summing the count of each name & race combination\n",
    "gdf = sdf.groupby(['name_last','race'], as_index=False)['race'].agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5261c6d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a pivot table so that each name has a count of the # of races with that last name\n",
    "gdf = gdf.pivot_table(values='count', columns='race',index='name_last')\n",
    "\n",
    "# Converting NaN to zeros since that means there is no one that identifies with that race with that last name\n",
    "gdf = gdf.fillna(0)\n",
    "\n",
    "# Getting the totals of each last name\n",
    "gdf['total_n'] = gdf.sum(axis=1)\n",
    "gdf.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "93304ee5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>name_last</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>multi_racial</th>\n",
       "      <th>native_indian</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0kharitonenko</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1amirthanayagam</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4r</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>77348 dancing rochanavibhata</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a de feria</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a de fernandez</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>a f r stephenson</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>a felix</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>a ghaffar</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>a malivert</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>a reys</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>a sanchez godoy</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>a z rodriguez</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>a'brunzo</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>a'hara</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "race                     name_last  asian  hispanic  multi_racial  \\\n",
       "0                    0kharitonenko    0.0       0.0           0.0   \n",
       "1                  1amirthanayagam    1.0       0.0           0.0   \n",
       "2                               4r    0.0       0.0           0.0   \n",
       "3     77348 dancing rochanavibhata    1.0       0.0           0.0   \n",
       "4                       a de feria    0.0       1.0           0.0   \n",
       "5                   a de fernandez    0.0       1.0           0.0   \n",
       "6                 a f r stephenson    0.0       0.0           0.0   \n",
       "7                          a felix    0.0       1.0           0.0   \n",
       "8                        a ghaffar    0.0       0.0           0.0   \n",
       "9                       a malivert    0.0       0.0           0.0   \n",
       "10                          a reys    0.0       0.0           0.0   \n",
       "11                 a sanchez godoy    0.0       1.0           0.0   \n",
       "12                   a z rodriguez    0.0       1.0           0.0   \n",
       "13                        a'brunzo    0.0       0.0           0.0   \n",
       "14                          a'hara    0.0       0.0           0.0   \n",
       "\n",
       "race  native_indian  nh_black  nh_white  other  total_n  \n",
       "0               0.0       0.0       1.0    0.0      1.0  \n",
       "1               0.0       0.0       0.0    0.0      1.0  \n",
       "2               0.0       0.0       1.0    0.0      1.0  \n",
       "3               0.0       0.0       0.0    0.0      1.0  \n",
       "4               0.0       0.0       0.0    0.0      1.0  \n",
       "5               0.0       0.0       0.0    0.0      1.0  \n",
       "6               0.0       0.0       1.0    0.0      1.0  \n",
       "7               0.0       0.0       0.0    0.0      1.0  \n",
       "8               0.0       0.0       1.0    0.0      1.0  \n",
       "9               0.0       1.0       0.0    0.0      1.0  \n",
       "10              0.0       1.0       0.0    0.0      1.0  \n",
       "11              0.0       0.0       0.0    0.0      1.0  \n",
       "12              0.0       0.0       0.0    0.0      1.0  \n",
       "13              0.0       0.0       2.0    0.0      2.0  \n",
       "14              0.0       0.0       3.0    0.0      3.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2f819e61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['asian',\n",
       " 'hispanic',\n",
       " 'multi_racial',\n",
       " 'native_indian',\n",
       " 'nh_black',\n",
       " 'nh_white',\n",
       " 'other']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "races = sorted(sdf.race.unique().tolist())\n",
    "races"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "58e1e7e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 43.7 ms, sys: 12 ms, total: 55.7 ms\n",
      "Wall time: 54.9 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>name_last</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>multi_racial</th>\n",
       "      <th>native_indian</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0kharitonenko</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1amirthanayagam</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4r</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>77348 dancing rochanavibhata</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a de feria</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868653</th>\n",
       "      <td>zyzanski</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868654</th>\n",
       "      <td>zyzdryn</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868655</th>\n",
       "      <td>zyznomyrsky</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868656</th>\n",
       "      <td>zzaman</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868657</th>\n",
       "      <td>zzie</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>868658 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race                       name_last  asian  hispanic  multi_racial  \\\n",
       "0                      0kharitonenko    0.0       0.0           0.0   \n",
       "1                    1amirthanayagam    1.0       0.0           0.0   \n",
       "2                                 4r    0.0       0.0           0.0   \n",
       "3       77348 dancing rochanavibhata    1.0       0.0           0.0   \n",
       "4                         a de feria    0.0       1.0           0.0   \n",
       "...                              ...    ...       ...           ...   \n",
       "868653                      zyzanski    0.0       0.0           0.0   \n",
       "868654                       zyzdryn    0.0       0.0           0.0   \n",
       "868655                   zyznomyrsky    0.0       0.0           0.0   \n",
       "868656                        zzaman    0.5       0.0           0.0   \n",
       "868657                          zzie    0.0       0.0           0.0   \n",
       "\n",
       "race    native_indian  nh_black  nh_white  other  total_n  \n",
       "0                 0.0       0.0       1.0    0.0      1.0  \n",
       "1                 0.0       0.0       0.0    0.0      1.0  \n",
       "2                 0.0       0.0       1.0    0.0      1.0  \n",
       "3                 0.0       0.0       0.0    0.0      1.0  \n",
       "4                 0.0       0.0       0.0    0.0      1.0  \n",
       "...               ...       ...       ...    ...      ...  \n",
       "868653            0.0       0.0       1.0    0.0      1.0  \n",
       "868654            0.0       0.0       1.0    0.0      1.0  \n",
       "868655            0.0       0.0       1.0    0.0      1.0  \n",
       "868656            0.0       0.0       0.0    0.5      1.0  \n",
       "868657            0.0       0.0       1.0    0.0      1.0  \n",
       "\n",
       "[868658 rows x 9 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Calculate the proportion of people with a particular last name\n",
    "#  that identify with various races/ethnicity\n",
    "gdf.iloc[:, 1:] = gdf.iloc[:, 1:].div(gdf.total_n, axis=0)\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f891a18d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_race_idx(val, races):\n",
    "    race_idx = races.index(val)\n",
    "    return race_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "75c86f60",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for one set of analysis, we define 'true race/ethincity' = where max prob (so modal race = true race)\n",
    "\n",
    "gdf['true_race'] = gdf[races].idxmax(axis=1)\n",
    "gdf['true_race'] = gdf['true_race'].apply(lambda c: get_race_idx(c,races))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d3c51335",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>name_last</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>multi_racial</th>\n",
       "      <th>native_indian</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "      <th>true_race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0kharitonenko</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1amirthanayagam</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4r</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>77348 dancing rochanavibhata</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a de feria</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868653</th>\n",
       "      <td>zyzanski</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868654</th>\n",
       "      <td>zyzdryn</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868655</th>\n",
       "      <td>zyznomyrsky</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868656</th>\n",
       "      <td>zzaman</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868657</th>\n",
       "      <td>zzie</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>868658 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race                       name_last  asian  hispanic  multi_racial  \\\n",
       "0                      0kharitonenko    0.0       0.0           0.0   \n",
       "1                    1amirthanayagam    1.0       0.0           0.0   \n",
       "2                                 4r    0.0       0.0           0.0   \n",
       "3       77348 dancing rochanavibhata    1.0       0.0           0.0   \n",
       "4                         a de feria    0.0       1.0           0.0   \n",
       "...                              ...    ...       ...           ...   \n",
       "868653                      zyzanski    0.0       0.0           0.0   \n",
       "868654                       zyzdryn    0.0       0.0           0.0   \n",
       "868655                   zyznomyrsky    0.0       0.0           0.0   \n",
       "868656                        zzaman    0.5       0.0           0.0   \n",
       "868657                          zzie    0.0       0.0           0.0   \n",
       "\n",
       "race    native_indian  nh_black  nh_white  other  total_n  true_race  \n",
       "0                 0.0       0.0       1.0    0.0      1.0          5  \n",
       "1                 0.0       0.0       0.0    0.0      1.0          0  \n",
       "2                 0.0       0.0       1.0    0.0      1.0          5  \n",
       "3                 0.0       0.0       0.0    0.0      1.0          0  \n",
       "4                 0.0       0.0       0.0    0.0      1.0          1  \n",
       "...               ...       ...       ...    ...      ...        ...  \n",
       "868653            0.0       0.0       1.0    0.0      1.0          5  \n",
       "868654            0.0       0.0       1.0    0.0      1.0          5  \n",
       "868655            0.0       0.0       1.0    0.0      1.0          5  \n",
       "868656            0.0       0.0       0.0    0.5      1.0          0  \n",
       "868657            0.0       0.0       1.0    0.0      1.0          5  \n",
       "\n",
       "[868658 rows x 10 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25cafb98",
   "metadata": {},
   "source": [
    "## Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4f65011a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(868658, 10)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proto_df = gdf.groupby('true_race', group_keys=False).apply(lambda x: x.sample(frac=1, random_state=10))\n",
    "proto_df.reset_index(inplace=True)\n",
    "proto_df.drop('index', axis=1, inplace=True)\n",
    "proto_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "059236e1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5    514516\n",
       "1    213834\n",
       "4     87831\n",
       "0     30058\n",
       "6     12600\n",
       "2      7538\n",
       "3      2281\n",
       "Name: true_race, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check that our sampling has generated proportionate representation in all classes\n",
    "proto_df.true_race.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "997167ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.48 s, sys: 47.5 ms, total: 6.52 s\n",
      "Wall time: 6.52 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# build n-gram list\n",
    "NGRAMS = 2\n",
    "vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=.005, ngram_range=(NGRAMS, NGRAMS), lowercase=False) \n",
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "a = vect.fit_transform(proto_df.name_last) \n",
    "tfidf = tfidf_transformer.fit_transform(a)\n",
    "\n",
    "vocab = vect.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "838ae1b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['er', 'rs', 'sa', 'ab', 'ba', 'al', 'pa', 'at', 'tt']\n",
      "num_words = 320\n"
     ]
    }
   ],
   "source": [
    "words = []\n",
    "for b in vocab:\n",
    "    c = vocab[b]\n",
    "    words.append((a[:, c].sum(), b))\n",
    "\n",
    "words_list = [w[1] for w in words]\n",
    "print(words_list[1:10])\n",
    "num_words = len(words_list)\n",
    "print(\"num_words = %d\" % num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "eb48932b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_ngrams(text, n):\n",
    "    a = zip(*[text[i:] for i in range(n)])\n",
    "    wi = []\n",
    "    for i in a:\n",
    "        w = ''.join(i)\n",
    "        try:\n",
    "            idx = words_list.index(w)\n",
    "        except:\n",
    "            idx = 0\n",
    "        wi.append(idx)\n",
    "    return wi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1900bd3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# build tf-idf vectors\n",
    "proto_df['tfidf_index'] = proto_df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "baab5ac9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>race</th>\n",
       "      <th>name_last</th>\n",
       "      <th>asian</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>multi_racial</th>\n",
       "      <th>native_indian</th>\n",
       "      <th>nh_black</th>\n",
       "      <th>nh_white</th>\n",
       "      <th>other</th>\n",
       "      <th>total_n</th>\n",
       "      <th>true_race</th>\n",
       "      <th>tfidf_index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bersabal</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pattana</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>patankar</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>kalapa</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pao</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.02381</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868653</th>\n",
       "      <td>maravall</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>868653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868654</th>\n",
       "      <td>tisoh</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>868654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868655</th>\n",
       "      <td>basanta perez</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>868655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868656</th>\n",
       "      <td>marcenaro vizquerra</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>868656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868657</th>\n",
       "      <td>baran-basit</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6</td>\n",
       "      <td>868657</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>868658 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "race              name_last     asian  hispanic  multi_racial  native_indian  \\\n",
       "0                  bersabal  0.625000  0.000000      0.000000            0.0   \n",
       "1                   pattana  1.000000  0.000000      0.000000            0.0   \n",
       "2                  patankar  0.714286  0.142857      0.142857            0.0   \n",
       "3                    kalapa  1.000000  0.000000      0.000000            0.0   \n",
       "4                       pao  0.428571  0.333333      0.000000            0.0   \n",
       "...                     ...       ...       ...           ...            ...   \n",
       "868653             maravall  0.000000  0.000000      0.000000            0.0   \n",
       "868654                tisoh  0.000000  0.000000      0.000000            0.0   \n",
       "868655        basanta perez  0.000000  0.000000      0.000000            0.0   \n",
       "868656  marcenaro vizquerra  0.000000  0.000000      0.000000            0.0   \n",
       "868657          baran-basit  0.000000  0.000000      0.000000            0.0   \n",
       "\n",
       "race    nh_black  nh_white     other  total_n  true_race  tfidf_index  \n",
       "0        0.00000  0.125000  0.250000      1.0          0            0  \n",
       "1        0.00000  0.000000  0.000000      1.0          0            1  \n",
       "2        0.00000  0.000000  0.000000      1.0          0            2  \n",
       "3        0.00000  0.000000  0.000000      1.0          0            3  \n",
       "4        0.02381  0.142857  0.071429      1.0          0            4  \n",
       "...          ...       ...       ...      ...        ...          ...  \n",
       "868653   0.00000  0.000000  1.000000      1.0          6       868653  \n",
       "868654   0.00000  0.000000  1.000000      1.0          6       868654  \n",
       "868655   0.00000  0.000000  1.000000      1.0          6       868655  \n",
       "868656   0.00000  0.000000  1.000000      1.0          6       868656  \n",
       "868657   0.00000  0.000000  1.000000      1.0          6       868657  \n",
       "\n",
       "[868658 rows x 11 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proto_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "df80f84e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: (783963, 12)\n",
      "Validation set size: (41262, 12)\n",
      "Test set size: (43433, 12)\n"
     ]
    }
   ],
   "source": [
    "train_df, test_df = train_test_split(proto_df, test_size=.05)\n",
    "train_df, valid_df = train_test_split(train_df, test_size=.05)\n",
    "\n",
    "train_df.reset_index(inplace=True)\n",
    "valid_df.reset_index(inplace=True)\n",
    "test_df.reset_index(inplace=True)\n",
    "\n",
    "print('Training set size: {}'.format(train_df.shape))\n",
    "print('Validation set size: {}'.format(valid_df.shape))\n",
    "print('Test set size: {}'.format(test_df.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab48a8a0",
   "metadata": {},
   "source": [
    "# Find the Best K\n",
    "i.e. the nearest k neighbors in our vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e80960b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def predict_race(arg):\n",
    "    # reading the tuple passed on by the calling function\n",
    "    idx, row_data, test_df, corpus_df, corp_vector, k, filt = arg\n",
    "\n",
    "    # resizing the tf-idf (1, m) & corpus vectors to be (n, m)\n",
    "    #  n = number of samples\n",
    "    #  m = number of dimentions\n",
    "    orig_vector = tfidf[row_data['tfidf_index']].reshape(1, -1)\n",
    "    # corp_vector = np.array([x for x in corpus_df['tfidf']])\n",
    "\n",
    "    # calculating the cosine similarity beteween the name vector\n",
    "    #   and the corpus vectors.  Then filtering for only values\n",
    "    #   that are greater that what was passed on\n",
    "    cossim = cosine_similarity(orig_vector, corp_vector)\n",
    "    filt_result = np.argwhere(cossim >= filt).reshape(-1)\n",
    "\n",
    "    # if we don't get any matches on cosine similarity >= \"value\"\n",
    "    #    we open up the critiria to 0.1 to get something\n",
    "    if (len(filt_result) == 0):\n",
    "        # this is to handle if we still are not getting anything\n",
    "        #  after opening up cosine similarity.  Just return the\n",
    "        #  most common class, which is nh_white/3\n",
    "        return 3\n",
    "\n",
    "    # filtering the corpus dataframe to only inclue the items\n",
    "    #   that met the cosine similarity filter\n",
    "    filtered_corpus_df = corpus_df.iloc[filt_result]\n",
    "\n",
    "    # calculate the levenshtein distance between our vector\n",
    "    #   and the filtered corpus vectors.\n",
    "    # Levenshtein is an expensive operation so we don't\n",
    "    #   want to calculate it for every name in the corpus\n",
    "    lev_dist = calc_leven_vector(row_data['name_last'],\n",
    "                          filtered_corpus_df['name_last'])\n",
    "\n",
    "    # The calc_leven function returns a dictionary\n",
    "    #  we seperate the keys from the values into arrays\n",
    "    #  that we can use which names are the most similar\n",
    "    #  i.e. smallest levenstein distance\n",
    "    values = np.array(list(lev_dist.values()))\n",
    "    keys = np.array(list(lev_dist.keys()))\n",
    "\n",
    "    if (k < values.shape[0]):\n",
    "        # This is when k is smaller than the size of the\n",
    "        #   values array, we can partition it by the smallest\n",
    "        #   k values\n",
    "        filt_values = np.argpartition(values, k)\n",
    "        max_value = np.max(values[filt_values[:k]])\n",
    "    else:\n",
    "        # Otherwise whatever the filt_value are will be the\n",
    "        #   k nearest neighbors to our string\n",
    "        filt_values = values.shape[0] - 1\n",
    "        max_value = np.max(values[filt_values])\n",
    "\n",
    "    # if (isinstance(filt_values, np.ndarray)):\n",
    "    #     max_value = np.max(values[filt_values[:k]])\n",
    "    # else:\n",
    "    #     max_value = np.max(values[filt_values])\n",
    "\n",
    "    mask = (values <= max_value) & (values > 0)\n",
    "    mask_idx = np.argwhere(mask).reshape(-1)\n",
    "    df_idx = keys[mask_idx]    \n",
    "    \n",
    "    filter_df = corpus_df.iloc[df_idx]\n",
    "\n",
    "    total_sum  = (filter_df['total_n'].sum())\n",
    "    \n",
    "    pred_asian    = (filter_df['asian'].dot(filter_df['total_n'])).sum() / total_sum\n",
    "    pred_hispanic = (filter_df['hispanic'].dot(filter_df['total_n'])).sum() / total_sum\n",
    "    pred_mr    = (filter_df['multi_racial'].dot(filter_df['total_n'])).sum() / total_sum\n",
    "    pred_ni    = (filter_df['native_indian'].dot(filter_df['total_n'])).sum() / total_sum\n",
    "    pred_black = (filter_df['nh_black'].dot(filter_df['total_n'])) / total_sum\n",
    "    pred_white = (filter_df['nh_white'].dot(filter_df['total_n'])).sum() / total_sum\n",
    "    pred_other = (filter_df['other'].dot(filter_df['total_n'])) / total_sum\n",
    "\n",
    "    predictions = [pred_asian, pred_hispanic, pred_mr, pred_ni, pred_black, pred_white, pred_other]\n",
    "\n",
    "    # final_pred.append(predictions.index(max(predictions)))\n",
    "    test_df.loc[test_df['name_last'] == row_data['name_last'],\n",
    "                'pred_race'] = predictions.index(max(predictions))\n",
    "\n",
    "    return predictions.index(max(predictions))\n",
    "\n",
    "\n",
    "def calc_leven(orig_string, filt_df):\n",
    "    lev_dist = {}\n",
    "    if not (isinstance(filt_df, str)):\n",
    "        for idx, row in filt_df.iteritems():\n",
    "            lev = lv.distance(orig_string, row)\n",
    "            lev_dist[idx] = lev\n",
    "    else:\n",
    "        lev = lv.distance(orig_string, filt_df)\n",
    "        lev_dist[0] = lev\n",
    "    return lev_dist\n",
    "\n",
    "def calc_leven_vector(orig_string, filt_df):\n",
    "    if not (isinstance(filt_df, str)):\n",
    "        lev_dist = filt_df.apply(lambda c: lv.distance(orig_string, c))\n",
    "        return lev_dist.to_dict()\n",
    "    else:\n",
    "        lev = lv.distance(orig_string, filt_df)\n",
    "        return {0: lev}\n",
    "\n",
    "def check_k(test_df, corpus_df, k, filt):\n",
    "    results = []\n",
    "\n",
    "    num_cpu = mp.cpu_count() \n",
    "    pool = mp.pool.ThreadPool(processes=8)\n",
    "\n",
    "    corp_vector = tfidf[corpus_df['tfidf_index']]\n",
    "\n",
    "    # for idx, row in tqdm(test_df.iterrows()):\n",
    "    r = pool.map(predict_race, [(idx, row, test_df, corpus_df, corp_vector, k, filt)\n",
    "                                for idx, row in test_df.iterrows()])\n",
    "    results.append(r)\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "630eaee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "k_metrics = {\n",
    "    3:0,\n",
    "    5:0,\n",
    "    25:0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7c1784f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert numbers to strings\n",
    "replacement = {0: 'asian', 1: 'hispanic', 2: 'multi_racial', 3: 'native_indian', 4: 'nh_black', 5: 'nh_white', 6: 'other'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f0a7e910",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_list = []\n",
    "for idx, row in valid_df.iterrows():\n",
    "    true_list.append(row['true_race'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "efabb525",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for value of k: 3 \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        asian       0.56      0.20      0.30      1416\n",
      "     hispanic       0.85      0.78      0.81     10112\n",
      " multi_racial       0.07      0.01      0.02       345\n",
      "native_indian       0.01      0.01      0.01       105\n",
      "     nh_black       0.59      0.42      0.49      4222\n",
      "     nh_white       0.79      0.91      0.85     24462\n",
      "        other       0.19      0.03      0.05       600\n",
      "\n",
      "     accuracy                           0.78     41262\n",
      "    macro avg       0.44      0.34      0.36     41262\n",
      " weighted avg       0.76      0.78      0.76     41262\n",
      "\n",
      "for value of k: 5 \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        asian       0.61      0.15      0.25      1416\n",
      "     hispanic       0.86      0.77      0.81     10112\n",
      " multi_racial       0.08      0.00      0.01       345\n",
      "native_indian       0.02      0.01      0.01       105\n",
      "     nh_black       0.61      0.40      0.48      4222\n",
      "     nh_white       0.78      0.93      0.85     24462\n",
      "        other       0.25      0.03      0.05       600\n",
      "\n",
      "     accuracy                           0.78     41262\n",
      "    macro avg       0.46      0.33      0.35     41262\n",
      " weighted avg       0.76      0.78      0.76     41262\n",
      "\n",
      "for value of k: 25 \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        asian       0.58      0.08      0.14      1416\n",
      "     hispanic       0.87      0.74      0.80     10112\n",
      " multi_racial       0.06      0.00      0.01       345\n",
      "native_indian       0.02      0.01      0.01       105\n",
      "     nh_black       0.62      0.31      0.41      4222\n",
      "     nh_white       0.76      0.94      0.84     24462\n",
      "        other       0.15      0.01      0.02       600\n",
      "\n",
      "     accuracy                           0.77     41262\n",
      "    macro avg       0.44      0.30      0.32     41262\n",
      " weighted avg       0.75      0.77      0.74     41262\n",
      "\n",
      "CPU times: user 3h 27min 20s, sys: 35min 21s, total: 4h 2min 42s\n",
      "Wall time: 1h 36min 29s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for value, key in enumerate (k_metrics):\n",
    "    #print ('{} -- {}'.format(key, value))\n",
    "    result = check_k(valid_df, train_df, key, 0.6)\n",
    "    \n",
    "    pred_list = np.array(result).reshape(-1)\n",
    "    pred_list = pred_list.tolist()\n",
    "    \n",
    "    true_list = pd.Series(true_list).replace(replacement).to_list()\n",
    "    pred_list = pd.Series(pred_list).replace(replacement).to_list()\n",
    "    \n",
    "    value = classification_report(true_list, pred_list, zero_division = 0)\n",
    "    \n",
    "    print ('for value of k: {} \\n{}'.format(key, value))\n",
    "    k_metrics[key] = value"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e318095e",
   "metadata": {},
   "source": [
    "# Test Set evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4f1c3f03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1h 9min 32s, sys: 11min 25s, total: 1h 20min 57s\n",
      "Wall time: 32min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "result = check_k(test_df, train_df, 3, 0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "234698b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "        asian       0.54      0.21      0.30      1504\n",
      "     hispanic       0.85      0.78      0.81     10775\n",
      " multi_racial       0.02      0.00      0.00       364\n",
      "native_indian       0.02      0.02      0.02       111\n",
      "     nh_black       0.59      0.42      0.49      4483\n",
      "     nh_white       0.79      0.92      0.85     25614\n",
      "        other       0.18      0.03      0.04       582\n",
      "\n",
      "     accuracy                           0.78     43433\n",
      "    macro avg       0.43      0.34      0.36     43433\n",
      " weighted avg       0.76      0.78      0.76     43433\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_list = np.array(result).reshape(-1)\n",
    "pred_list = pred_list.tolist()\n",
    "\n",
    "true_list = []\n",
    "for idx, row in test_df.iterrows():\n",
    "    true_list.append(row['true_race'])\n",
    "\n",
    "true_list = pd.Series(true_list).replace(replacement).to_list()\n",
    "pred_list = pd.Series(pred_list).replace(replacement).to_list()\n",
    "\n",
    "value = classification_report(true_list, pred_list)\n",
    "\n",
    "print (value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}