CPU_demo_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9f69f6e5-29f9-4e17-a9ac-3b07f526d841",
   "metadata": {},
   "source": [
    "# Start Dask Cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6d01787a",
   "metadata": {
    "executionInfo": {
     "elapsed": 3708,
     "status": "ok",
     "timestamp": 1649137487780,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "6d01787a"
   },
   "outputs": [],
   "source": [
    "from dask.distributed import Client\n",
    "from dask.distributed import LocalCluster\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "73e6c9d4-d55c-4aa4-8f5c-0f5a53f77c7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f\"rm -rf ./dask-worker-space\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2fe57bb8-9e4c-4468-8b96-0262d3c3001c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the dask dashboard can be found here:\n",
      "http://127.0.0.1:8787/status\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cluster = LocalCluster(silence_logs=50)\n",
    "client = Client(cluster)\n",
    "print(f\"the dask dashboard can be found here:\\n{client.dashboard_link}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90df20ed-9c10-4852-8f36-089486592bd6",
   "metadata": {},
   "source": [
    "## **A note on Dask:**\n",
    "Dask uses '**lazy evaluation**.'  \n",
    "This means that Dask will not calculate any of the processes performed on a Dask DataFrame (or other Dask object) until the **.compute()** method is explicitly called.  \n",
    "Any write function performed by dask (e.g. the .to_csv() method) automatically calls the .compute() method, unless otherwise specified.  \n",
    "Normally, .compute() method is rarely called, if ever.  \n",
    "  \n",
    "In this experiment, the .compute() method is called often for the sole purpose of calculating the time it takes to run a certain process; Dask code is not usually written this way.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a3b21b9-2860-41ee-a167-35885f54b003",
   "metadata": {},
   "source": [
    "# Import other libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cf8f9957-e9e5-407e-a552-cc55922a2f28",
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as pd\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.cluster import KMeans, DBSCAN\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from dask_ml.xgboost import XGBClassifier\n",
    "from xgboost.dask import DaskDMatrix\n",
    "import xgboost as xgb\n",
    "from sklearn.decomposition import TruncatedSVD, PCA\n",
    "import numpy as np\n",
    "from dask_ml.preprocessing import LabelEncoder\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5fc9697",
   "metadata": {
    "id": "f5fc9697",
    "tags": []
   },
   "source": [
    "# Input filename\n",
    "Adjust the filename to one of the files available in the sample_data folder.  \n",
    "Files are availble publicly for download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2c0fa4f3",
   "metadata": {
    "executionInfo": {
     "elapsed": 276,
     "status": "ok",
     "timestamp": 1649137560831,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "2c0fa4f3"
   },
   "outputs": [],
   "source": [
    "in_filename = 'sample_data_20m.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47228ef1",
   "metadata": {
    "id": "47228ef1",
    "tags": []
   },
   "source": [
    "# Define variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "2ab38848",
   "metadata": {
    "executionInfo": {
     "elapsed": 384,
     "status": "ok",
     "timestamp": 1649137561998,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "2ab38848"
   },
   "outputs": [],
   "source": [
    "label_col_name = 'is_counterfeit' # the name of the column that contain the labels (0 for negative, 1 for positive)\n",
    "\n",
    "predictor_col_names = ['Size of U.S. market', # the names of the columns that will be used to predict the label\n",
    "                       'Price per unit', \n",
    "                       'RX/OTC', \n",
    "                       'Indication',\n",
    "                       'Drug Class',\n",
    "                       'Shortage',\n",
    "                       'Twitter Mentions'\n",
    "                      ] \n",
    "\n",
    "cetegoric_predictor_col_names = ['RX/OTC', # list of predictor columns that are not values; will use a label encoder on these\n",
    "                                 'Indication',\n",
    "                                 'Drug Class', \n",
    "                                 'Shortage'\n",
    "                                ]\n",
    "\n",
    "# models perform better when values are between 0 and 1. \n",
    "# thus, we use a scaling value to attempt to do this\n",
    "scaling_dict = {'Size of U.S. market': 1/100000000000, # each column in this dictionary will be multiplied by the values here\n",
    "                'Price per unit': 1/10000,\n",
    "                'Twitter Mentions': 1/200000\n",
    "               }\n",
    "\n",
    "val_set_percent = 0.1 # percent of data to be reserved for validation\n",
    "test_set_percent = 0.1 # percent of data to be reserved for testing\n",
    "\n",
    "chunksize = \"100MB\"\n",
    "blocksize = \"100MB\"\n",
    "\n",
    "random_seed = 42 # to make results reproducible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "41d0f28d",
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1649137562412,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "41d0f28d"
   },
   "outputs": [],
   "source": [
    "out_filename = 'write_speed_test_gpu.csv'\n",
    "in_filename = './sample_data/' + in_filename\n",
    "out_filename = './output_data/' + out_filename"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82bc6c4b-1e04-422e-a499-09e9846a4bd6",
   "metadata": {},
   "source": [
    "# Define class for temp file names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "28379b94-950f-473f-b35d-67ba35685795",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TempFileNameSwitcher:\n",
    "    def __init__(self):\n",
    "        self.temp_names = ['./output_data/temp_1.csv', './output_data/temp_2.csv']\n",
    "        self.cur_iter = 0\n",
    "        self.name = self.temp_names[self.cur_iter]\n",
    "        for a_name in self.temp_names:\n",
    "            try:\n",
    "                os.system(f\"rm -rf {a_name}\")\n",
    "            except:\n",
    "                pass\n",
    "            try:\n",
    "                os.system(f\"rm -f {a_name}\")\n",
    "            except:\n",
    "                pass\n",
    "        \n",
    "    def switch(self):\n",
    "        self.cur_iter += 1\n",
    "        name_i = self.cur_iter % len(self.temp_names)\n",
    "        self.name = self.temp_names[name_i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e001ba10-ad7e-4e3d-b1c7-a1c539d8fafc",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_file_name_switcher = TempFileNameSwitcher()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d76f4d83-a6b8-494f-b1de-d368ee5e5709",
   "metadata": {},
   "source": [
    "# ETL"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1168b27b-8878-4469-87f0-73d5de5902c4",
   "metadata": {},
   "source": [
    "## Single CSV file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32ec91fb-40fa-495e-a969-8317c9b382c9",
   "metadata": {},
   "source": [
    "### read"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "55159c21-73ab-4323-bf45-aeef7cbcd30c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(in_filename, blocksize=blocksize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0ffb7e16-ef33-44ed-8f29-e0dc294771a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.56 s, sys: 3.07 s, total: 7.63 s\n",
      "Wall time: 10.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = df.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4f1bd94-86a1-4ed0-ba8f-853c6b909d18",
   "metadata": {},
   "source": [
    "### write"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "287f6e4a-58d6-442e-a2b4-7a7451ffc1ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f\"rm -rf {out_filename}\")\n",
    "os.system(f\"rm -f {out_filename}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5e281865-be66-438b-b6ff-3f78840f7796",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.5 s, sys: 1.73 s, total: 9.23 s\n",
      "Wall time: 1min 16s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "_ = df.to_csv(out_filename, index=False, single_file=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c592f6fa-ed5e-4b08-a247-8b5d7d11568b",
   "metadata": {},
   "source": [
    "## Multiple CSV files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "567c64f6-eb9e-4db9-a0ce-4ae66edf8460",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f\"rm -rf {out_filename}\")\n",
    "os.system(f\"rm -f {out_filename}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e7e2570-ec7d-4052-a585-fad76e3e8e0b",
   "metadata": {},
   "source": [
    "### write"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "993c2d98-f2e0-423b-b17d-4897e0ce9a74",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.09 s, sys: 475 ms, total: 2.56 s\n",
      "Wall time: 18.7 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/00.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/01.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/02.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/03.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/04.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/05.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/06.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/07.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/08.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/09.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/10.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/11.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/12.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/13.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/14.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/15.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/16.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/17.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/18.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/19.part',\n",
       " '/home/jcosme/proj/GPU_vs_CPU_v2/output_data/write_speed_test_gpu.csv/20.part']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "_ = df.to_csv(out_filename, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03249aa1-7d3f-4889-8af2-9312190288b8",
   "metadata": {},
   "source": [
    "### read"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "89929f82-d959-40c4-a6d9-0fbc3b5dd81f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(f\"{out_filename}/*\", blocksize=blocksize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "72792320-7d65-460f-945f-a52e65ce5a0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.71 s, sys: 3.16 s, total: 7.87 s\n",
      "Wall time: 10.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = df.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3dbb602-8397-46a7-a657-b426cf5c1e6d",
   "metadata": {
    "id": "f3dbb602-8397-46a7-a657-b426cf5c1e6d"
   },
   "source": [
    "## Common DataFrame Operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2cac3ed2-e090-43ec-919a-4d1723f0dfd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(in_filename, blocksize=blocksize)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "173b4d5b-6659-4a19-959e-7acd8cc4d197",
   "metadata": {
    "id": "173b4d5b-6659-4a19-959e-7acd8cc4d197"
   },
   "source": [
    "### describe the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "5bfc39f8-7ffc-43d7-aecb-ed2779d5c5e6",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 335
    },
    "executionInfo": {
     "elapsed": 1717,
     "status": "ok",
     "timestamp": 1649137949938,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "5bfc39f8-7ffc-43d7-aecb-ed2779d5c5e6",
    "outputId": "d2bb465b-aa11-4af4-bb4f-64084d77f0b3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.05 s, sys: 202 ms, total: 1.25 s\n",
      "Wall time: 3.76 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_counterfeit</th>\n",
       "      <th>Size of U.S. market</th>\n",
       "      <th>Price per unit</th>\n",
       "      <th>Twitter Mentions</th>\n",
       "      <th>SNOMED</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>2.000000e+07</td>\n",
       "      <td>2.000000e+07</td>\n",
       "      <td>2.000000e+07</td>\n",
       "      <td>2.000000e+07</td>\n",
       "      <td>2.000000e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.800000e-01</td>\n",
       "      <td>2.319031e+09</td>\n",
       "      <td>2.073848e+02</td>\n",
       "      <td>9.099047e+03</td>\n",
       "      <td>2.584983e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>4.853865e-01</td>\n",
       "      <td>3.181265e+09</td>\n",
       "      <td>3.072970e+02</td>\n",
       "      <td>5.702964e+03</td>\n",
       "      <td>2.758066e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.002100e+04</td>\n",
       "      <td>1.000000e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>7.321101e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.998876e+07</td>\n",
       "      <td>2.540000e+00</td>\n",
       "      <td>6.249000e+03</td>\n",
       "      <td>8.065901e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.743000e+09</td>\n",
       "      <td>6.750000e+01</td>\n",
       "      <td>1.061800e+04</td>\n",
       "      <td>1.192920e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>7.259273e+09</td>\n",
       "      <td>6.904100e+02</td>\n",
       "      <td>1.688200e+04</td>\n",
       "      <td>3.736210e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.999998e+09</td>\n",
       "      <td>1.000000e+03</td>\n",
       "      <td>2.000000e+04</td>\n",
       "      <td>8.403580e+08</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       is_counterfeit  Size of U.S. market  Price per unit  Twitter Mentions  \\\n",
       "count    2.000000e+07         2.000000e+07    2.000000e+07      2.000000e+07   \n",
       "mean     3.800000e-01         2.319031e+09    2.073848e+02      9.099047e+03   \n",
       "std      4.853865e-01         3.181265e+09    3.072970e+02      5.702964e+03   \n",
       "min      0.000000e+00         1.002100e+04    1.000000e-01      0.000000e+00   \n",
       "25%      0.000000e+00         4.998876e+07    2.540000e+00      6.249000e+03   \n",
       "50%      1.000000e+00         1.743000e+09    6.750000e+01      1.061800e+04   \n",
       "75%      1.000000e+00         7.259273e+09    6.904100e+02      1.688200e+04   \n",
       "max      1.000000e+00         9.999998e+09    1.000000e+03      2.000000e+04   \n",
       "\n",
       "             SNOMED  \n",
       "count  2.000000e+07  \n",
       "mean   2.584983e+08  \n",
       "std    2.758066e+08  \n",
       "min    7.321101e+07  \n",
       "25%    8.065901e+07  \n",
       "50%    1.192920e+08  \n",
       "75%    3.736210e+08  \n",
       "max    8.403580e+08  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "_ = df.describe().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14e786b7-2525-43e0-b5bf-6535a0947f2d",
   "metadata": {
    "id": "14e786b7-2525-43e0-b5bf-6535a0947f2d"
   },
   "source": [
    "### Set Index\n",
    "for each categorical variable, set the DataFrame index to that variable\n",
    "  \n",
    "**note:** dask dataframes have a 'set_index' method, but it is slower than applying the 'set_index' method to each partition (so we will use the latter method)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "52e89f27-668f-4dd8-b2d5-6085818e3979",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_cat_index(df, a_cat):\n",
    "    return df.set_index(a_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "25292546-1479-43f6-935c-d57919347fe2",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 724,
     "status": "ok",
     "timestamp": 1649137952069,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "25292546-1479-43f6-935c-d57919347fe2",
    "outputId": "fd17010c-128a-4350-b348-bede10542176"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RX/OTC\n",
      "Indication\n",
      "Drug Class\n",
      "Shortage\n",
      "CPU times: user 17.7 s, sys: 10.8 s, total: 28.5 s\n",
      "Wall time: 41.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for a_cat in cetegoric_predictor_col_names:\n",
    "    print(a_cat)\n",
    "    _ = df.map_partitions(set_cat_index, a_cat).compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43cfa654-f85f-4064-b57f-6e82a80803d7",
   "metadata": {
    "id": "43cfa654-f85f-4064-b57f-6e82a80803d7"
   },
   "source": [
    "### Concat multiple DataFrames\n",
    "concatenate 3 copies of the DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "169837d8-3e91-4bb3-8f1c-a95614cc4bd1",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 295,
     "status": "ok",
     "timestamp": 1649137954690,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "169837d8-3e91-4bb3-8f1c-a95614cc4bd1",
    "outputId": "58cfaaf7-ef38-4c66-a6e5-632fb1e35768"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12.7 s, sys: 8.78 s, total: 21.5 s\n",
      "Wall time: 26.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = pd.concat([df] * 3, ignore_index=True).compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b86df6a3-1539-4c8e-9249-91ffdc698d6a",
   "metadata": {
    "id": "b86df6a3-1539-4c8e-9249-91ffdc698d6a",
    "tags": []
   },
   "source": [
    "### Groupby function\n",
    "for each value column, we calculate the mean by grouping by each categorical variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "91a4cd1b-2e14-46ef-ba3c-41d3987bcf1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Twitter Mentions', 'Price per unit', 'Size of U.S. market']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_cetegoric_predictor_col_names = list(set(predictor_col_names).difference(set(cetegoric_predictor_col_names)))\n",
    "non_cetegoric_predictor_col_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "52471a46-3083-4e1d-bdaa-b7a71eee10d2",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 627,
     "status": "ok",
     "timestamp": 1649137959268,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "52471a46-3083-4e1d-bdaa-b7a71eee10d2",
    "outputId": "4e718eb8-19a3-4037-9d49-3fa7131e5e43"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RX/OTC\n",
      "\tTwitter Mentions\n",
      "\tPrice per unit\n",
      "\tSize of U.S. market\n",
      "Indication\n",
      "\tTwitter Mentions\n",
      "\tPrice per unit\n",
      "\tSize of U.S. market\n",
      "Drug Class\n",
      "\tTwitter Mentions\n",
      "\tPrice per unit\n",
      "\tSize of U.S. market\n",
      "Shortage\n",
      "\tTwitter Mentions\n",
      "\tPrice per unit\n",
      "\tSize of U.S. market\n",
      "CPU times: user 7.07 s, sys: 1.49 s, total: 8.56 s\n",
      "Wall time: 37.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for a_cat in cetegoric_predictor_col_names:\n",
    "    print(a_cat)\n",
    "    for a_col in non_cetegoric_predictor_col_names:\n",
    "        print(f\"\\t{a_col}\")\n",
    "        _ = df.groupby(a_cat)[a_col].mean().compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f57f1437",
   "metadata": {
    "id": "f57f1437"
   },
   "source": [
    "## Preprocess data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "c3825c96",
   "metadata": {
    "executionInfo": {
     "elapsed": 253,
     "status": "ok",
     "timestamp": 1649137961416,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "c3825c96"
   },
   "outputs": [],
   "source": [
    "df_input = df.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a283b33",
   "metadata": {
    "id": "5a283b33"
   },
   "source": [
    "### fit label encoder\n",
    "first we create a label encoder for each column specified in the variable 'cetegoric_predictor_col_names' \n",
    "defined at the top of the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "9ecf777b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2993,
     "status": "ok",
     "timestamp": 1649137967139,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "9ecf777b",
    "outputId": "3270c21f-a761-4547-efed-f2ec8e53595a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RX/OTC\n",
      "Indication\n",
      "Shortage\n",
      "Drug Class\n",
      "CPU times: user 3.04 s, sys: 615 ms, total: 3.66 s\n",
      "Wall time: 19.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cats_dict = {}\n",
    "for col in df_input.columns:\n",
    "    if col in cetegoric_predictor_col_names:\n",
    "        print(col)\n",
    "        cats_dict[col] = LabelEncoder()\n",
    "        cats_dict[col].fit(df_input[col])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ad8819a",
   "metadata": {
    "id": "0ad8819a"
   },
   "source": [
    "### encode categoric columns\n",
    "Then we apply the label encoder to the values of those columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "394fd6af",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 4390,
     "status": "ok",
     "timestamp": 1649137973367,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "394fd6af",
    "outputId": "a48ecb53-4fee-43a6-c582-daffbc24567e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RX/OTC\n",
      "Indication\n",
      "Shortage\n",
      "Drug Class\n",
      "CPU times: user 2.18 s, sys: 499 ms, total: 2.68 s\n",
      "Wall time: 13.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for col in df_input.columns:\n",
    "    if col in cetegoric_predictor_col_names:\n",
    "        print(col)\n",
    "        df_input[col] = cats_dict[col].transform(df_input[col])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46aae733-8e7c-4848-b8e2-f296e9b07e1d",
   "metadata": {},
   "source": [
    "we save a temporary file, and reload the data from this file, so we can isolate the computation time of the next transformation (scale value variable columns).  \n",
    "Notice we do NOT save the csv file as a 'single file.' This operation is faster, but it means that we will have multiple files, instead of one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "db049731-ea6f-448b-b9d1-0d3d39b2667c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.45 s, sys: 569 ms, total: 3.02 s\n",
      "Wall time: 19.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "old_name = temp_file_name_switcher.name\n",
    "temp_file_name_switcher.switch()\n",
    "_ = df_input.to_csv(temp_file_name_switcher.name, index=False)\n",
    "df_input = pd.read_csv(f\"{temp_file_name_switcher.name}/*\")\n",
    "os.system(f\"rm -rf {old_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "427d4065",
   "metadata": {
    "id": "427d4065"
   },
   "source": [
    "### scale value variable columns\n",
    "Next we apply the scaling by the variable 'scaling_dict' defined at the top of the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "484305f2",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 254,
     "status": "ok",
     "timestamp": 1649137977628,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "484305f2",
    "outputId": "f244b43a-a861-441f-9eb8-5100637e272b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.8 s, sys: 2.57 s, total: 5.37 s\n",
      "Wall time: 7.76 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for col in scaling_dict.keys():\n",
    "    df_input[col] = df_input[col] * scaling_dict[col]\n",
    "_ = df_input.compute()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4c4d2b1-1c24-40e1-bb1c-51c48da6a119",
   "metadata": {},
   "source": [
    "we save a temporary file, and reload the data from this file, so we can isolate the computation time of the next transformation (train/val/test split).  \n",
    "Notice we do NOT save the csv file as a 'single file.' This operation is faster, but it means that we will have multiple files, instead of one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f0508612-06a5-4a00-be34-0998b5251593",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.41 s, sys: 529 ms, total: 2.94 s\n",
      "Wall time: 18.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "old_name = temp_file_name_switcher.name\n",
    "temp_file_name_switcher.switch()\n",
    "_ = df_input.to_csv(temp_file_name_switcher.name, index=False)\n",
    "df_input = pd.read_csv(f\"{temp_file_name_switcher.name}/*\")\n",
    "os.system(f\"rm -rf {old_name}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bccf873f-15d4-4cb0-9cfc-93bc7851071a",
   "metadata": {},
   "source": [
    "## train/val/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "0fea2c9a-faa7-40cf-96b7-5a51ff379c3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "not_train_prct = val_set_percent + test_set_percent\n",
    "train_prct = 1. - not_train_prct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "0d63492b-eafa-4f91-af91-8b02b78cda12",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_val, df_test = df_input.random_split([train_prct, val_set_percent, test_set_percent], random_state=random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "4d1aa5a4-7755-422f-b168-efe8037dbaad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.41 s, sys: 2.77 s, total: 7.18 s\n",
      "Wall time: 13.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = df_train.compute()\n",
    "_ = df_val.compute()\n",
    "_ = df_test.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "111ef8e2",
   "metadata": {
    "executionInfo": {
     "elapsed": 256,
     "status": "ok",
     "timestamp": 1649137981499,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "111ef8e2"
   },
   "outputs": [],
   "source": [
    "X_train = df_train[predictor_col_names].copy().astype(float)\n",
    "y_train = df_train[label_col_name].copy().astype(float)\n",
    "\n",
    "X_val = df_val[predictor_col_names].copy().astype(float)\n",
    "y_val = df_val[label_col_name].copy().astype(float)\n",
    "\n",
    "X_test = df_test[predictor_col_names].copy().astype(float)\n",
    "y_test = df_test[label_col_name].copy().astype(float)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e167626e-ebb5-40a4-89f7-07767f8e25c8",
   "metadata": {},
   "source": [
    "we save temporary files for each split, and reload them from this file, so we can isolate the computation time of the next operations (dimensionality reduction & model fitting)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "3360dd58-54d8-4ef8-94cb-07c38d6d8a2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.system(f\"rm -rf ./output_data/X_train\")\n",
    "os.system(f\"rm -rf ./output_data/y_train\")\n",
    "\n",
    "os.system(f\"rm -rf ./output_data/X_val\")\n",
    "os.system(f\"rm -rf ./output_data/y_val\")\n",
    "\n",
    "os.system(f\"rm -rf ./output_data/X_test\")\n",
    "os.system(f\"rm -rf ./output_data/y_test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "ca3ecde5-8246-4eab-ac3b-731e7745e805",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.52 s, sys: 1.11 s, total: 6.63 s\n",
      "Wall time: 34.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "_ = X_train.to_csv('./output_data/X_train', index=False)\n",
    "_ = y_train.to_frame().to_csv('./output_data/y_train', index=False)\n",
    "\n",
    "_ = X_val.to_csv('./output_data/X_val', index=False)\n",
    "_ = y_val.to_frame().to_csv('./output_data/y_val', index=False)\n",
    "\n",
    "_ = X_test.to_csv('./output_data/X_test', index=False)\n",
    "_ = y_test.to_frame().to_csv('./output_data/y_test', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "be3bad9a",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1134,
     "status": "ok",
     "timestamp": 1649137983888,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "be3bad9a",
    "outputId": "5c04e764-2621-4b6d-b73b-d044eaab6ca6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36.9 ms, sys: 3.78 ms, total: 40.7 ms\n",
      "Wall time: 36.9 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_train = pd.read_csv('./output_data/X_train/*')\n",
    "y_train = pd.read_csv('./output_data/y_train/*')\n",
    "y_train = y_train[y_train.columns[0]]\n",
    "\n",
    "X_val = pd.read_csv('./output_data/X_val/*')\n",
    "y_val = pd.read_csv('./output_data/y_val/*')\n",
    "y_val = y_val[y_val.columns[0]]\n",
    "\n",
    "X_test = pd.read_csv('./output_data/X_test/*')\n",
    "y_test = pd.read_csv('./output_data/y_test/*')\n",
    "y_test = y_test[y_test.columns[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "1a446021-ab19-4994-b9c1-6e9236c06ee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "del df, df_input"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a4fee11-3a1c-4657-a7b6-6166c0c7c838",
   "metadata": {},
   "source": [
    "# Dimensionality Reduction (2D)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72db5672-09cb-4c57-9645-a7b280694800",
   "metadata": {},
   "source": [
    "## PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "7abef570-f72d-49cf-b921-9af6c1a24f7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 23s, sys: 1min 9s, total: 2min 33s\n",
      "Wall time: 14.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with joblib.parallel_backend('dask'):\n",
    "    pca = PCA(n_components=2, random_state=random_seed)\n",
    "    X_train_pca = pca.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "60f6d6e5-e4cf-4f22-8ba2-51eea1db9911",
   "metadata": {},
   "outputs": [],
   "source": [
    "del pca, X_train_pca"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be5f63f0-a278-47b1-8341-eb2cc40bd3c3",
   "metadata": {},
   "source": [
    "## TruncatedSVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "b613b52f-fe27-4d49-ae98-ba965601978f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 35s, sys: 1min 19s, total: 2min 54s\n",
      "Wall time: 14.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with joblib.parallel_backend('dask'):\n",
    "    trunSVD = TruncatedSVD(n_components=2, random_state=random_seed)\n",
    "    X_train_trunSVD = trunSVD.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "1bf56fa6-0f34-4022-81d1-659c0dcc801c",
   "metadata": {},
   "outputs": [],
   "source": [
    "del trunSVD, X_train_trunSVD"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27250faa",
   "metadata": {
    "id": "27250faa"
   },
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a7c080c",
   "metadata": {
    "id": "5a7c080c"
   },
   "source": [
    "## OLS Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "d73eb090",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1437,
     "status": "ok",
     "timestamp": 1649137999472,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "d73eb090",
    "outputId": "4f655caa-5348-4c37-a7ad-fc53f360c8c7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.83 s, sys: 17.6 s, total: 25.4 s\n",
      "Wall time: 5.17 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with joblib.parallel_backend('dask'):\n",
    "    reg = LinearRegression()\n",
    "    reg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "57b94fac-fc40-4951-895a-b0d519ab6e2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "del reg"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a764282",
   "metadata": {
    "id": "2a764282"
   },
   "source": [
    "## K-Means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "069ea71f-d797-48ee-8db2-ecebb0f19a46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_clusters = y_train.unique().compute().shape[0]\n",
    "n_clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "395fe7b3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2839,
     "status": "ok",
     "timestamp": 1649138034847,
     "user": {
      "displayName": "Jonathan Cosme",
      "userId": "03866196755566426705"
     },
     "user_tz": 300
    },
    "id": "395fe7b3",
    "outputId": "049c9512-dab2-4bfa-ae47-794e48de83af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 1s, sys: 1min, total: 2min 2s\n",
      "Wall time: 24.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with joblib.parallel_backend('dask'):\n",
    "    kmeans = KMeans(n_clusters=n_clusters, random_state=random_seed)\n",
    "    kmeans.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "6a0c3230-8063-4799-b162-c8036bf58f87",
   "metadata": {},
   "outputs": [],
   "source": [
    "del kmeans"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "406d2cf6-eb92-423a-8697-0e3fca0c7996",
   "metadata": {},
   "source": [
    "## Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "761bbea7-351a-4baf-b098-5beb1947f13f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 184 ms, sys: 62.8 ms, total: 246 ms\n",
      "Wall time: 763 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "dtrain = DaskDMatrix(client, X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "bb20698c-0d16-4444-b56c-e31c49f3a249",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/xgboost/dask.py:884: RuntimeWarning: coroutine 'Client._wait_for_workers' was never awaited\n",
      "  client.wait_for_workers(n_workers)\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "/home/jcosme/miniconda3/envs/gvc-2/lib/python3.9/site-packages/distributed/worker_state_machine.py:3649: FutureWarning: The `Worker.nthreads` attribute has been moved to `Worker.state.nthreads`\n",
      "  warnings.warn(\n",
      "[15:02:11] task [xgboost.dask-0]:tcp://127.0.0.1:35419 got new rank 0\n",
      "[15:02:11] task [xgboost.dask-1]:tcp://127.0.0.1:37861 got new rank 1\n",
      "[15:02:11] task [xgboost.dask-2]:tcp://127.0.0.1:36941 got new rank 2\n",
      "[15:02:11] task [xgboost.dask-3]:tcp://127.0.0.1:43349 got new rank 3\n",
      "[15:02:11] task [xgboost.dask-4]:tcp://127.0.0.1:34821 got new rank 4\n",
      "[15:02:12] task [xgboost.dask-5]:tcp://127.0.0.1:38523 got new rank 5\n",
      "[15:02:12] task [xgboost.dask-6]:tcp://127.0.0.1:36505 got new rank 6\n",
      "[15:02:12] task [xgboost.dask-7]:tcp://127.0.0.1:45155 got new rank 7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 679 ms, sys: 205 ms, total: 884 ms\n",
      "Wall time: 4.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "xgb_model = xgb.dask.train(\n",
    "        client,\n",
    "    {\"tree_method\": \"hist\", \"random_state\": random_seed},\n",
    "    dtrain,\n",
    "    \n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "9870a3f2-8690-4f72-9f59-a67a3ffe619e",
   "metadata": {},
   "outputs": [],
   "source": [
    "del xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5be5fca3-48cf-4ce2-8473-63fe749dede3",
   "metadata": {},
   "outputs": [],
   "source": [
    "client.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e9e282f0-d9a0-4806-b488-366070837964",
   "metadata": {},
   "outputs": [],
   "source": [
    "del client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82ccc097-8747-47e8-a7cb-6d0b10d1b32c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "GPU Counterfeit Drug Detection.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "gvc-2",
   "language": "python",
   "name": "gvc-2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}