notebooks/1.0_rf_rapids_fullname_unigrams.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1FD6iTh_-5WDH6T842WQ376n-i3ZAY9bp","timestamp":1684270343611}],"gpuType":"A100","machine_shape":"hm","mount_file_id":"1FD6iTh_-5WDH6T842WQ376n-i3ZAY9bp","authorship_tag":"ABX9TyMoKYn2olobTubBQ1WLGmbU"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","gpuClass":"standard"},"cells":[{"cell_type":"markdown","source":["## Installing RAPIDS"],"metadata":{"id":"qDpDSjNfV11T"}},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"QU4KDF3xxKAm","executionInfo":{"status":"ok","timestamp":1684116614202,"user_tz":420,"elapsed":2055,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"30fd1822-2f27-44b0-aaa8-d0db2602c408"},"outputs":[{"output_type":"stream","name":"stdout","text":["Cloning into 'rapidsai-csp-utils'...\n","remote: Enumerating objects: 390, done.\u001b[K\n","remote: Counting objects: 100% (121/121), done.\u001b[K\n","remote: Compressing objects: 100% (70/70), done.\u001b[K\n","remote: Total 390 (delta 89), reused 51 (delta 51), pack-reused 269\u001b[K\n","Receiving objects: 100% (390/390), 107.11 KiB | 5.64 MiB/s, done.\n","Resolving deltas: 100% (191/191), done.\n","PLEASE READ FOR 21.06\n","********************************************************************************************************\n","Another release, another script change.  We had to revise the script, which now:\n","1. Does a more comprehensive install\n","2. Includes BlazingSQL\n","3. is far easier for everyone to understand and maintain\n","\n","The script will require you to add these 5 cells to your notebook.  We have also created a new startup template: \n","https://colab.research.google.com/drive/1TAAi_szMfWqRfHVfjGSqnGVLr_ztzUM9?usp=sharing\n","\n","CHANGES T\n","CELL 1:\n","    # This get the RAPIDS-Colab install files and test check your GPU.  Run cells 1 and 2 only.\n","    # Please read the output of this cell.  If your Colab Instance is not RAPIDS compatible, it will warn you and give you remediation steps.\n","    !git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n","    !python rapidsai-csp-utils/colab/env-check.py\n","\n","CELL 2:\n","    # This will update the Colab environment and restart the kernel.\n","    !bash rapidsai-csp-utils/colab/update_gcc.sh\n","    import os\n","    os._exit(00)\n","\n","CELL 3:\n","    ## Installing CondaColab.  This will restart your kernel again\n","    import condacolab\n","    condacolab.install()\n","\n","CELL 4:\n","    import condacolab\n","    condacolab.check()\n","\n","CELL 5:\n","    # Installing RAPIDS is now 'python rapidsai-csp-utils/colab/install_rapids.py <release> <packages>'\n","    # The <release> options are 'stable' and 'nightly'.  Leaving it blank or adding any other words will default to stable.\n","    # The <packages> option are default blank or 'core'.  By default, we install RAPIDSAI and BlazingSQL.  The 'core' option will install only RAPIDSAI and not include BlazingSQL, \n","    !python rapidsai-csp-utils/colab/install_rapids.py nightly\n","    import os\n","    os.environ['NUMBAPRO_NVVM'] = '/usr/local/cuda/nvvm/lib64/libnvvm.so'\n","    os.environ['NUMBAPRO_LIBDEVICE'] = '/usr/local/cuda/nvvm/libdevice/'\n","    os.environ['CONDA_PREFIX'] = '/usr/local'\n","\n","********************************************************************************************************\n","\n","Enjoy using RAPIDS!  If you have any issues with or suggestions for RAPIDSAI on Colab, please create a issue on https://github.com/rapidsai/rapidsai-csp-utils/issues/new.\n"]}],"source":["!git clone https://github.com/rapidsai/rapidsai-csp-utils.git\n","!bash rapidsai-csp-utils/colab/rapids-colab.sh stable"]},{"cell_type":"code","source":["!python --version"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZjHx9WGwxwbY","executionInfo":{"status":"ok","timestamp":1684116615790,"user_tz":420,"elapsed":1027,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"95a3ab65-8bc6-428f-b00f-8324b700d596"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Python 3.10.11\n"]}]},{"cell_type":"code","source":["!python rapidsai-csp-utils/colab/env-check.py"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ZpFkJ8Ckx2la","executionInfo":{"status":"ok","timestamp":1684116624094,"user_tz":420,"elapsed":5058,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"d2312dd1-f04b-4753-d340-3892a6d8383a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting pynvml\n","  Downloading pynvml-11.5.0-py3-none-any.whl (53 kB)\n","     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.1/53.1 kB 6.1 MB/s eta 0:00:00\n","Installing collected packages: pynvml\n","Successfully installed pynvml-11.5.0\n","***********************************************************************\n","Woo! Your instance has the right kind of GPU, a NVIDIA A100-SXM4-40GB!\n","We will now install RAPIDS via pip!  Please stand by, should be quick...\n","***********************************************************************\n","\n"]}]},{"cell_type":"code","source":["!bash rapidsai-csp-utils/colab/update_gcc.sh\n","import os\n","os._exit(00)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"i8gopVCKynJo","outputId":"54f521ea-6361-49fc-8c4a-263aa93d203b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Updating your Colab environment.  This will restart your kernel.  Don't Panic!\n","Found existing installation: cupy-cuda11x 11.0.0\n","Uninstalling cupy-cuda11x-11.0.0:\n","  Successfully uninstalled cupy-cuda11x-11.0.0\n","Get:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu focal InRelease [18.1 kB]\n","Get:2 https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/ InRelease [3,622 B]\n","Hit:3 http://ppa.launchpad.net/cran/libgit2/ubuntu focal InRelease\n","Hit:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n","Hit:5 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal InRelease\n","Get:6 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu focal InRelease [24.3 kB]\n","Get:7 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal InRelease [17.5 kB]\n","Hit:8 http://archive.ubuntu.com/ubuntu focal InRelease\n","Get:9 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB]\n","Hit:10 http://ppa.launchpad.net/ubuntugis/ppa/ubuntu focal InRelease\n","Get:11 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu focal/main Sources [2,576 kB]\n","Get:12 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB]\n","Get:13 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu focal/main amd64 Packages [1,215 kB]\n","Get:14 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB]\n","Get:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu focal/main amd64 Packages [46.8 kB]\n","Get:16 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal/main amd64 Packages [19.1 kB]\n","Get:17 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,681 kB]\n","Get:18 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,343 kB]\n","Get:19 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [3,158 kB]\n","Get:20 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [1,046 kB]\n","Fetched 12.5 MB in 2s (6,425 kB/s)\n","Reading package lists... Done\n","Added repo\n","Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease\n","Hit:2 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu focal InRelease\n","Hit:3 https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/ InRelease\n","Hit:4 http://ppa.launchpad.net/cran/libgit2/ubuntu focal InRelease\n","Hit:5 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal InRelease\n","Hit:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n","Hit:7 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu focal InRelease\n","Hit:8 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal InRelease\n","Hit:9 http://ppa.launchpad.net/ubuntugis/ppa/ubuntu focal InRelease\n","Hit:10 http://archive.ubuntu.com/ubuntu focal InRelease\n","Hit:11 http://archive.ubuntu.com/ubuntu focal-updates InRelease\n","Hit:12 http://archive.ubuntu.com/ubuntu focal-backports InRelease\n","Reading package lists... Done\n","Installing libstdc++\n","Reading package lists... Done\n","Building dependency tree       \n","Reading state information... Done\n","Selected version '11.1.0-1ubuntu1~20.04' (Toolchain test builds:20.04/focal [amd64]) for 'libstdc++6'\n","The following additional packages will be installed:\n","  gcc-11-base\n","The following NEW packages will be installed:\n","  gcc-11-base\n","The following packages will be upgraded:\n","  libstdc++6\n","1 upgraded, 1 newly installed, 0 to remove and 50 not upgraded.\n","Need to get 603 kB of archives.\n","After this operation, 510 kB of additional disk space will be used.\n","Get:1 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal/main amd64 gcc-11-base amd64 11.1.0-1ubuntu1~20.04 [19.0 kB]\n","Get:2 http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu focal/main amd64 libstdc++6 amd64 11.1.0-1ubuntu1~20.04 [584 kB]\n","Fetched 603 kB in 0s (5,205 kB/s)\n","debconf: unable to initialize frontend: Dialog\n","debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 2.)\n","debconf: falling back to frontend: Readline\n","debconf: unable to initialize frontend: Readline\n","debconf: (This frontend requires a controlling tty.)\n","debconf: falling back to frontend: Teletype\n","dpkg-preconfigure: unable to re-open stdin: \n","Selecting previously unselected package gcc-11-base:amd64.\n","(Reading database ... 122519 files and directories currently installed.)\n","Preparing to unpack .../gcc-11-base_11.1.0-1ubuntu1~20.04_amd64.deb ...\n","Unpacking gcc-11-base:amd64 (11.1.0-1ubuntu1~20.04) ...\n","Setting up gcc-11-base:amd64 (11.1.0-1ubuntu1~20.04) ...\n","(Reading database ... 122524 files and directories currently installed.)\n","Preparing to unpack .../libstdc++6_11.1.0-1ubuntu1~20.04_amd64.deb ...\n","Unpacking libstdc++6:amd64 (11.1.0-1ubuntu1~20.04) over (10.3.0-1ubuntu1~20.04) ...\n","Setting up libstdc++6:amd64 (11.1.0-1ubuntu1~20.04) ...\n","Processing triggers for libc-bin (2.31-0ubuntu9.9) ...\n","restarting Colab...\n"]}]},{"cell_type":"code","source":["## Installing CondaColab.  This will restart your kernel again\n","import condacolab\n","condacolab.install()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8Pz8se49yu8W","executionInfo":{"status":"ok","timestamp":1684116663181,"user_tz":420,"elapsed":8331,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"bc8356b2-b230-4a97-bda3-0ce864344520"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["⏬ Downloading https://github.com/conda-forge/miniforge/releases/download/23.1.0-1/Mambaforge-23.1.0-1-Linux-x86_64.sh...\n","📦 Installing...\n","📌 Adjusting configuration...\n","🩹 Patching environment...\n","⏲ Done in 0:00:08\n","🔁 Restarting kernel...\n"]}]},{"cell_type":"code","source":["import condacolab\n","condacolab.check()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"JNkvi7SCzQNe","executionInfo":{"status":"ok","timestamp":1684116673966,"user_tz":420,"elapsed":4,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"a420b586-f04d-4fbf-d799-5d369907be5b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["✨🍰✨ Everything looks OK!\n"]}]},{"cell_type":"code","source":["!python rapidsai-csp-utils/colab/install_rapids.py stable\n","\n","import os\n","os.environ['NUMBAPRO_NVVM'] = '/usr/local/cuda/nvvm/lib64/libnvvm.so'\n","os.environ['NUMBAPRO_LIBDEVICE'] = '/usr/local/cuda/nvvm/libdevice/'\n","os.environ['CONDA_PREFIX'] = '/usr/local'"],"metadata":{"id":"0VLONjAuxaVa"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Load data using cudf"],"metadata":{"id":"HpTWFD5AWLrC"}},{"cell_type":"code","source":["import cudf as pd\n"],"metadata":{"id":"UVZG39gxyCIS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df = pd.read_csv('/content/drive/MyDrive/Colab/ethnicolor/data/fl_reg_name_race_2022.csv.gz')"],"metadata":{"id":"ty8QKXIT2Hc5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"tU2-uMrS2Ubx","executionInfo":{"status":"ok","timestamp":1684129045151,"user_tz":420,"elapsed":20,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"98a46ebd-3bea-41e5-eaa7-a82a5d59e773"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["       name_last name_first      race\n","0  Hessler-Smith      Jason  nh_white\n","1         Rogers      Renee  nh_white\n","2      Bartolome    Crystal  nh_white\n","3         Bailey      Donna  nh_white\n","4        Carlson   Greggory  nh_white"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","      <th>name_first</th>\n","      <th>race</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Hessler-Smith</td>\n","      <td>Jason</td>\n","      <td>nh_white</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Rogers</td>\n","      <td>Renee</td>\n","      <td>nh_white</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Bartolome</td>\n","      <td>Crystal</td>\n","      <td>nh_white</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Bailey</td>\n","      <td>Donna</td>\n","      <td>nh_white</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Carlson</td>\n","      <td>Greggory</td>\n","      <td>nh_white</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["df.describe()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":268},"id":"oIAWt8TR2WkO","executionInfo":{"status":"ok","timestamp":1684129045152,"user_tz":420,"elapsed":19,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"7c8bc11e-ca0b-45a0-f405-2f46e5590f73"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/site-packages/cudf/core/dataframe.py:4953: FutureWarning: `datetime_is_numeric` is deprecated. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.\n","  warnings.warn(\n","/usr/local/lib/python3.10/site-packages/cudf/core/series.py:3120: FutureWarning: `datetime_is_numeric` is deprecated and will be removed in a future release. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.\n","  warnings.warn(\n"]},{"output_type":"execute_result","data":{"text/plain":["       name_last name_first      race\n","count   15455009   15455080  15455110\n","unique   1341197     641105         8\n","top        Smith    Michael  nh_white\n","freq       79362     153753   9446851"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","      <th>name_first</th>\n","      <th>race</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>15455009</td>\n","      <td>15455080</td>\n","      <td>15455110</td>\n","    </tr>\n","    <tr>\n","      <th>unique</th>\n","      <td>1341197</td>\n","      <td>641105</td>\n","      <td>8</td>\n","    </tr>\n","    <tr>\n","      <th>top</th>\n","      <td>Smith</td>\n","      <td>Michael</td>\n","      <td>nh_white</td>\n","    </tr>\n","    <tr>\n","      <th>freq</th>\n","      <td>79362</td>\n","      <td>153753</td>\n","      <td>9446851</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":4}]},{"cell_type":"code","source":["df['race'].unique()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"e7QqjeJw2Xzv","executionInfo":{"status":"ok","timestamp":1684129045153,"user_tz":420,"elapsed":15,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"441e70e4-9176-4e7d-ef50-00b37edc834c"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0            asian\n","1         hispanic\n","2     multi_racial\n","3    native_indian\n","4         nh_black\n","5         nh_white\n","6            other\n","7          unknown\n","Name: race, dtype: object"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","source":["df.dropna(subset=['name_first', 'name_last'], inplace=True)"],"metadata":{"id":"8U0RhVkt2cBO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.describe()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":268},"id":"NQkkPRFb2kd7","executionInfo":{"status":"ok","timestamp":1684129046304,"user_tz":420,"elapsed":8,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"dcb007aa-49f9-4c64-9a28-77cfa77c893c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/site-packages/cudf/core/dataframe.py:4953: FutureWarning: `datetime_is_numeric` is deprecated. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.\n","  warnings.warn(\n","/usr/local/lib/python3.10/site-packages/cudf/core/series.py:3120: FutureWarning: `datetime_is_numeric` is deprecated and will be removed in a future release. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.\n","  warnings.warn(\n"]},{"output_type":"execute_result","data":{"text/plain":["       name_last name_first      race\n","count   15454979   15454979  15454979\n","unique   1341194     641100         8\n","top        Smith    Michael  nh_white\n","freq       79362     153753   9446764"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","      <th>name_first</th>\n","      <th>race</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>15454979</td>\n","      <td>15454979</td>\n","      <td>15454979</td>\n","    </tr>\n","    <tr>\n","      <th>unique</th>\n","      <td>1341194</td>\n","      <td>641100</td>\n","      <td>8</td>\n","    </tr>\n","    <tr>\n","      <th>top</th>\n","      <td>Smith</td>\n","      <td>Michael</td>\n","      <td>nh_white</td>\n","    </tr>\n","    <tr>\n","      <th>freq</th>\n","      <td>79362</td>\n","      <td>153753</td>\n","      <td>9446764</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["df.groupby('race').agg({'name_last': 'nunique'})"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":332},"id":"7P-Nx01s2mwz","executionInfo":{"status":"ok","timestamp":1684129048148,"user_tz":420,"elapsed":273,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"df65cd83-161d-412f-f35e-37427ae4fbfe"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["               name_last\n","race                    \n","asian              81743\n","hispanic          442817\n","multi_racial       37859\n","native_indian      23064\n","nh_black          162943\n","nh_white          802901\n","other             105674\n","unknown           143439"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","    </tr>\n","    <tr>\n","      <th>race</th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>asian</th>\n","      <td>81743</td>\n","    </tr>\n","    <tr>\n","      <th>hispanic</th>\n","      <td>442817</td>\n","    </tr>\n","    <tr>\n","      <th>multi_racial</th>\n","      <td>37859</td>\n","    </tr>\n","    <tr>\n","      <th>native_indian</th>\n","      <td>23064</td>\n","    </tr>\n","    <tr>\n","      <th>nh_black</th>\n","      <td>162943</td>\n","    </tr>\n","    <tr>\n","      <th>nh_white</th>\n","      <td>802901</td>\n","    </tr>\n","    <tr>\n","      <th>other</th>\n","      <td>105674</td>\n","    </tr>\n","    <tr>\n","      <th>unknown</th>\n","      <td>143439</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","source":["df['full_name'] = df['name_last'] + ' ' + df['name_first']"],"metadata":{"id":"86e6_4492p_u"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"t3xeUnCm2r8C","executionInfo":{"status":"ok","timestamp":1684129057033,"user_tz":420,"elapsed":6,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"3ee7d831-fdc0-4fed-c7ff-b47c16077a68"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["       name_last name_first      race            full_name\n","0  Hessler-Smith      Jason  nh_white  Hessler-Smith Jason\n","1         Rogers      Renee  nh_white         Rogers Renee\n","2      Bartolome    Crystal  nh_white    Bartolome Crystal\n","3         Bailey      Donna  nh_white         Bailey Donna\n","4        Carlson   Greggory  nh_white     Carlson Greggory"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","      <th>name_first</th>\n","      <th>race</th>\n","      <th>full_name</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Hessler-Smith</td>\n","      <td>Jason</td>\n","      <td>nh_white</td>\n","      <td>Hessler-Smith Jason</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Rogers</td>\n","      <td>Renee</td>\n","      <td>nh_white</td>\n","      <td>Rogers Renee</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Bartolome</td>\n","      <td>Crystal</td>\n","      <td>nh_white</td>\n","      <td>Bartolome Crystal</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Bailey</td>\n","      <td>Donna</td>\n","      <td>nh_white</td>\n","      <td>Bailey Donna</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Carlson</td>\n","      <td>Greggory</td>\n","      <td>nh_white</td>\n","      <td>Carlson Greggory</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":10}]},{"cell_type":"markdown","source":["## Get features using CountVectorizer"],"metadata":{"id":"s36stEj_WQUm"}},{"cell_type":"code","source":["from cuml.feature_extraction.text import CountVectorizer"],"metadata":{"id":"ilOE_rx82uXh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import cuml"],"metadata":{"id":"LaDmqajKVuD9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["cuml.global_settings.output_type"],"metadata":{"id":"T76ermn_Y-IQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["vectorizer = CountVectorizer(analyzer='char')"],"metadata":{"id":"5KQnsBJ42zHV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["X = vectorizer.fit_transform(df['full_name'])"],"metadata":{"id":"iMVydjua235J"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["X[:10].toarray()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"KTaTWl3h25r7","executionInfo":{"status":"ok","timestamp":1684121165324,"user_tz":420,"elapsed":730,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"ca9d118f-6a0c-4822-cdb3-3fb9d39f4f21"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n","        0., 0., 0., 2., 0., 0., 2., 1., 1., 0., 1., 1., 1., 1., 0., 0.,\n","        1., 4., 1., 0., 0., 0., 0., 0., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 4., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 0., 0.,\n","        3., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 2.,\n","        1., 1., 0., 1., 0., 0., 0., 0., 0., 0., 2., 1., 0., 2., 0., 0.,\n","        2., 1., 2., 0., 0., 0., 0., 1., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 2.,\n","        1., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 0., 2., 1., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n","        0., 1., 0., 1., 0., 3., 0., 0., 0., 0., 1., 0., 1., 2., 0., 0.,\n","        3., 1., 0., 0., 0., 0., 0., 1., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n","        0., 2., 1., 2., 0., 0., 0., 1., 0., 0., 2., 1., 0., 1., 0., 0.,\n","        2., 0., 1., 0., 0., 1., 0., 0., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 2., 0., 2., 1., 0., 0.,\n","        0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 2.,\n","        0., 2., 0., 2., 0., 0., 1., 1., 0., 1., 1., 0., 1., 0., 0., 0.,\n","        2., 0., 1., 0., 0., 0., 0., 0., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n","        0., 1., 1., 2., 0., 0., 2., 1., 0., 0., 1., 1., 0., 1., 0., 0.,\n","        1., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n","       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n","        0., 0., 0., 2., 0., 0., 0., 2., 1., 0., 0., 0., 2., 0., 1., 0.,\n","        0., 2., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)"]},"metadata":{},"execution_count":74}]},{"cell_type":"code","source":["vectorizer.get_feature_names()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bzUyxTx327Jm","executionInfo":{"status":"ok","timestamp":1684121169862,"user_tz":420,"elapsed":328,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"87518bf0-1903-405a-8fa9-3af3fb477c98"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0      \n","1     \"\n","2     #\n","3     %\n","4     '\n","5     (\n","6     )\n","7     *\n","8     +\n","9     ,\n","10    -\n","11    .\n","12    /\n","13    0\n","14    1\n","15    2\n","16    3\n","17    4\n","18    5\n","19    6\n","20    7\n","21    8\n","22    ;\n","23    =\n","24    ?\n","25    @\n","26    [\n","27    \\\n","28    ]\n","29    _\n","30    `\n","31    a\n","32    b\n","33    c\n","34    d\n","35    e\n","36    f\n","37    g\n","38    h\n","39    i\n","40    j\n","41    k\n","42    l\n","43    m\n","44    n\n","45    o\n","46    p\n","47    q\n","48    r\n","49    s\n","50    t\n","51    u\n","52    v\n","53    w\n","54    x\n","55    y\n","56    z\n","57    ~\n","Name: token, dtype: object"]},"metadata":{},"execution_count":75}]},{"cell_type":"code","source":["len(vectorizer.get_feature_names())"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"66nOHW4u2-dd","executionInfo":{"status":"ok","timestamp":1684121181122,"user_tz":420,"elapsed":674,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"0b50e1b0-2e04-40ea-fb43-deec7a57ec55"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["58"]},"metadata":{},"execution_count":76}]},{"cell_type":"code","source":["vocab = vectorizer.get_feature_names()"],"metadata":{"id":"3i8hAZWo2_t9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"rAl_tSuGa7fB","executionInfo":{"status":"ok","timestamp":1684121186401,"user_tz":420,"elapsed":456,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"2c83ddc7-f5d6-47fa-d965-9d4c3c07f0d4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["       name_last name_first      race            full_name  race_code\n","0  Hessler-Smith      Jason  nh_white  Hessler-Smith Jason          5\n","1         Rogers      Renee  nh_white         Rogers Renee          5\n","2      Bartolome    Crystal  nh_white    Bartolome Crystal          5\n","3         Bailey      Donna  nh_white         Bailey Donna          5\n","4        Carlson   Greggory  nh_white     Carlson Greggory          5"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","      <th>name_first</th>\n","      <th>race</th>\n","      <th>full_name</th>\n","      <th>race_code</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Hessler-Smith</td>\n","      <td>Jason</td>\n","      <td>nh_white</td>\n","      <td>Hessler-Smith Jason</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Rogers</td>\n","      <td>Renee</td>\n","      <td>nh_white</td>\n","      <td>Rogers Renee</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Bartolome</td>\n","      <td>Crystal</td>\n","      <td>nh_white</td>\n","      <td>Bartolome Crystal</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Bailey</td>\n","      <td>Donna</td>\n","      <td>nh_white</td>\n","      <td>Bailey Donna</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Carlson</td>\n","      <td>Greggory</td>\n","      <td>nh_white</td>\n","      <td>Carlson Greggory</td>\n","      <td>5</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":78}]},{"cell_type":"code","source":["X[0].toarray()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"14VUXm-UcoC6","executionInfo":{"status":"ok","timestamp":1684121239908,"user_tz":420,"elapsed":702,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"a4d3c08b-f250-4c3c-f528-74fbaab06471"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,\n","        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n","        0., 0., 0., 2., 0., 0., 2., 1., 1., 0., 1., 1., 1., 1., 0., 0.,\n","        1., 4., 1., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)"]},"metadata":{},"execution_count":81}]},{"cell_type":"code","source":["import cupy as np\n","\n","np.unique(df.race.factorize()[0])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"z5vTj5oRGPAj","executionInfo":{"status":"ok","timestamp":1684121368217,"user_tz":420,"elapsed":459,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"21527641-1f53-469e-c64c-0af825f1bd4b"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([0, 1, 2, 3, 4, 5, 6, 7], dtype=int8)"]},"metadata":{},"execution_count":82}]},{"cell_type":"code","source":["df['race_code'] = df.race.factorize()[0]"],"metadata":{"id":"qIK2QdEZ3EeJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"bSN9xLM93GG-","executionInfo":{"status":"ok","timestamp":1684121371429,"user_tz":420,"elapsed":6,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"5c34b5b7-83ec-4695-bfea-043dc5cb998c"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["       name_last name_first      race            full_name  race_code\n","0  Hessler-Smith      Jason  nh_white  Hessler-Smith Jason          5\n","1         Rogers      Renee  nh_white         Rogers Renee          5\n","2      Bartolome    Crystal  nh_white    Bartolome Crystal          5\n","3         Bailey      Donna  nh_white         Bailey Donna          5\n","4        Carlson   Greggory  nh_white     Carlson Greggory          5"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>name_last</th>\n","      <th>name_first</th>\n","      <th>race</th>\n","      <th>full_name</th>\n","      <th>race_code</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Hessler-Smith</td>\n","      <td>Jason</td>\n","      <td>nh_white</td>\n","      <td>Hessler-Smith Jason</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Rogers</td>\n","      <td>Renee</td>\n","      <td>nh_white</td>\n","      <td>Rogers Renee</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Bartolome</td>\n","      <td>Crystal</td>\n","      <td>nh_white</td>\n","      <td>Bartolome Crystal</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Bailey</td>\n","      <td>Donna</td>\n","      <td>nh_white</td>\n","      <td>Bailey Donna</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Carlson</td>\n","      <td>Greggory</td>\n","      <td>nh_white</td>\n","      <td>Carlson Greggory</td>\n","      <td>5</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":84}]},{"cell_type":"code","source":["df[['race', 'race_code']].drop_duplicates().sort_values('race_code')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":300},"id":"cP1s5x4T3Hty","executionInfo":{"status":"ok","timestamp":1684121372009,"user_tz":420,"elapsed":6,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"f8a6c750-faac-49ee-8c8d-f0520af3a8c4"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["              race  race_code\n","32           asian          0\n","22        hispanic          1\n","158   multi_racial          2\n","66   native_indian          3\n","5         nh_black          4\n","0         nh_white          5\n","6            other          6\n","120        unknown          7"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>race</th>\n","      <th>race_code</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>32</th>\n","      <td>asian</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>hispanic</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>158</th>\n","      <td>multi_racial</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>66</th>\n","      <td>native_indian</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>nh_black</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <th>0</th>\n","      <td>nh_white</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>other</td>\n","      <td>6</td>\n","    </tr>\n","    <tr>\n","      <th>120</th>\n","      <td>unknown</td>\n","      <td>7</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":85}]},{"cell_type":"code","source":["race_id_df = df[['race', 'race_code']].drop_duplicates().sort_values('race_code')"],"metadata":{"id":"T822m3_d3JNS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["race_id_df"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":300},"id":"xhqP8TL9HLup","executionInfo":{"status":"ok","timestamp":1684121377017,"user_tz":420,"elapsed":5,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"9e78bbc5-739a-47a8-c455-20bdd34f10c1"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["              race  race_code\n","32           asian          0\n","22        hispanic          1\n","158   multi_racial          2\n","66   native_indian          3\n","5         nh_black          4\n","0         nh_white          5\n","6            other          6\n","120        unknown          7"],"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>race</th>\n","      <th>race_code</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>32</th>\n","      <td>asian</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>hispanic</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>158</th>\n","      <td>multi_racial</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>66</th>\n","      <td>native_indian</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>nh_black</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <th>0</th>\n","      <td>nh_white</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>other</td>\n","      <td>6</td>\n","    </tr>\n","    <tr>\n","      <th>120</th>\n","      <td>unknown</td>\n","      <td>7</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"]},"metadata":{},"execution_count":87}]},{"cell_type":"code","source":["race_to_id = {}\n","id_to_race = {}\n","\n","for index, row in race_id_df.to_pandas().iterrows():\n","    race_to_id[row['race']] = row['race_code']\n","    id_to_race[row['race_code']] = row['race']"],"metadata":{"id":"fMU2xf03HHTR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["race_to_id"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DFRUgc9LJAly","executionInfo":{"status":"ok","timestamp":1684121381386,"user_tz":420,"elapsed":5,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"402d1cdd-20e9-415c-d0cc-346aa5f3bb69"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{'asian': 0,\n"," 'hispanic': 1,\n"," 'multi_racial': 2,\n"," 'native_indian': 3,\n"," 'nh_black': 4,\n"," 'nh_white': 5,\n"," 'other': 6,\n"," 'unknown': 7}"]},"metadata":{},"execution_count":89}]},{"cell_type":"code","source":["id_to_race"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NZl9LJXLJIw0","executionInfo":{"status":"ok","timestamp":1684121382748,"user_tz":420,"elapsed":3,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"a6af188d-a841-4506-e745-db4ab64d419a"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["{0: 'asian',\n"," 1: 'hispanic',\n"," 2: 'multi_racial',\n"," 3: 'native_indian',\n"," 4: 'nh_black',\n"," 5: 'nh_white',\n"," 6: 'other',\n"," 7: 'unknown'}"]},"metadata":{},"execution_count":90}]},{"cell_type":"code","source":["y = df.race_code.to_cupy()"],"metadata":{"id":"iqY75DNY3MUL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["X = X.toarray()"],"metadata":{"id":"QXWnk8KZJVDa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["y[0]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BHoxhYMp3Pr9","executionInfo":{"status":"ok","timestamp":1684121395769,"user_tz":420,"elapsed":518,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"7fb4c149-9f61-4bfc-d640-3cd291db9f1c"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(5, dtype=int8)"]},"metadata":{},"execution_count":93}]},{"cell_type":"code","source":["y.shape"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"s746Q3fpKboT","executionInfo":{"status":"ok","timestamp":1684117916413,"user_tz":420,"elapsed":8,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"d677871d-4659-400f-c8eb-bdfa6adf004f"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(15454979,)"]},"metadata":{},"execution_count":34}]},{"cell_type":"code","source":["type(X)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Yi0uR1BwVC9r","executionInfo":{"status":"ok","timestamp":1684119904609,"user_tz":420,"elapsed":419,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"fb254b3b-13b0-487b-eef2-45eae72953f6"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["cupyx.scipy.sparse._csr.csr_matrix"]},"metadata":{},"execution_count":56}]},{"cell_type":"markdown","source":["## Preparing for training"],"metadata":{"id":"RDDfEX3hWcKS"}},{"cell_type":"code","source":["from cuml.model_selection import train_test_split\n","\n","X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)"],"metadata":{"id":"zfzD5nb_3Rjm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from cuml.ensemble import RandomForestClassifier"],"metadata":{"id":"wFighRpoi6L8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["## Training"],"metadata":{"id":"8ratmBtWWhp1"}},{"cell_type":"code","source":["#Create a Gaussian Classifier\n","clf = RandomForestClassifier(split_criterion='entropy', random_state=42, verbose=True)\n","#Train the model using the training sets y_pred=clf.predict(X_test)\n","clf.fit(X_train, y_train)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"NuPCXVmhi8ec","executionInfo":{"status":"ok","timestamp":1684123130477,"user_tz":420,"elapsed":40039,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"a750f3cd-1101-4a27-abc2-8d5e6264750b"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/site-packages/cuml/internals/api_decorators.py:344: UserWarning: For reproducible results in Random Forest Classifier or for almost reproducible results in Random Forest Regressor, n_streams=1 is recommended. If n_streams is > 1, results may vary due to stream/thread timing differences, even when random_state is set\n","  return func(**kwargs)\n"]},{"output_type":"execute_result","data":{"text/plain":["RandomForestClassifier()"]},"metadata":{},"execution_count":98}]},{"cell_type":"markdown","source":["## Accuracy using test dataset"],"metadata":{"id":"u0fXAEmQWlab"}},{"cell_type":"code","source":["y_pred = clf.predict(X_test)\n","\n","# Model Accuracy, how often is the classifier correct?\n","print(\"Accuracy:\", cuml.metrics.accuracy.accuracy_score(y_test, y_pred))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cUne0E4NjNnL","executionInfo":{"status":"ok","timestamp":1684123307094,"user_tz":420,"elapsed":2088,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"8b2af9a5-0067-4cb6-c21e-5e2b196ef514"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Accuracy: 0.6747448444366455\n"]}]},{"cell_type":"markdown","source":["## Save model and vectorizer"],"metadata":{"id":"y7T4kO0rWsyZ"}},{"cell_type":"code","source":["import joblib\n","\n","joblib.dump(clf, \"/content/drive/MyDrive/Colab/ethnicolor/fl_voter_name_2022_rf_model_1g.joblib\", compress=3)  # compression is ON!\n","joblib.dump(vectorizer, \"/content/drive/MyDrive/Colab/ethnicolor/fl_voter_name_2022_rf_vect_1g.joblib\", compress=3)  # compression is ON!"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"uqOoh5cdmv81","executionInfo":{"status":"ok","timestamp":1684124115351,"user_tz":420,"elapsed":5947,"user":{"displayName":"Rajashekar Chintalapati","userId":"03596288833202137831"}},"outputId":"c4201a7c-a720-4097-b6eb-d1ea54509a0c"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['/content/drive/MyDrive/Colab/ethnicolor/fl_voter_name_2022_rf_vect_1g.joblib']"]},"metadata":{},"execution_count":107}]}]}