3_topic_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1e80acd3-bc80-4c78-9461-dd699a7bea8e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:16:48.497722Z",
     "iopub.status.busy": "2022-05-30T18:16:48.497494Z",
     "iopub.status.idle": "2022-05-30T18:17:25.896338Z",
     "shell.execute_reply": "2022-05-30T18:17:25.895634Z",
     "shell.execute_reply.started": "2022-05-30T18:16:48.497697Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47ec1bea336e463f990ea092a1f0ab5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Spark application\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>1</td><td>application_1653926235028_0002</td><td>pyspark</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-17-92.ec2.internal:20888/proxy/application_1653926235028_0002/\" >Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-18-198.ec2.internal:8042/node/containerlogs/container_1653926235028_0002_01_000001/livy\" >Link</a></td><td>✔</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SparkSession available as 'spark'.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                    Version   \n",
      "-------------------------- ----------\n",
      "awscli                     1.24.10   \n",
      "beautifulsoup4             4.9.3     \n",
      "boto                       2.49.0    \n",
      "botocore                   1.26.10   \n",
      "click                      7.1.2     \n",
      "colorama                   0.4.4     \n",
      "docutils                   0.16      \n",
      "jmespath                   0.10.0    \n",
      "joblib                     0.17.0    \n",
      "lxml                       4.6.1     \n",
      "mysqlclient                1.4.2     \n",
      "nltk                       3.5       \n",
      "nose                       1.3.4     \n",
      "numpy                      1.16.5    \n",
      "pip                        9.0.1     \n",
      "py-dateutil                2.2       \n",
      "pyasn1                     0.4.8     \n",
      "python-dateutil            2.8.2     \n",
      "python37-sagemaker-pyspark 1.4.1     \n",
      "pytz                       2020.1    \n",
      "PyYAML                     5.3.1     \n",
      "regex                      2020.10.28\n",
      "rsa                        4.7.2     \n",
      "s3transfer                 0.5.2     \n",
      "setuptools                 28.8.0    \n",
      "six                        1.13.0    \n",
      "spark-nlp                  3.4.4     \n",
      "tqdm                       4.51.0    \n",
      "urllib3                    1.26.9    \n",
      "wheel                      0.29.0    \n",
      "windmill                   1.6"
     ]
    }
   ],
   "source": [
    "sc.list_packages()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "93235740-7923-4bfa-972d-9997cbf0a472",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:17:25.897931Z",
     "iopub.status.busy": "2022-05-30T18:17:25.897677Z",
     "iopub.status.idle": "2022-05-30T18:17:59.307679Z",
     "shell.execute_reply": "2022-05-30T18:17:59.307007Z",
     "shell.execute_reply.started": "2022-05-30T18:17:25.897896Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa1b85104bd14f3ba0b447b322c6c8f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting boto3\n",
      "  Using cached https://files.pythonhosted.org/packages/75/ca/d917b244919f1ebf96f7bbd5a00e4641f7e9191b0d070258f5dc10f5eaad/boto3-1.23.10-py3-none-any.whl\n",
      "Requirement already satisfied: s3transfer<0.6.0,>=0.5.0 in /usr/local/lib/python3.7/site-packages (from boto3)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.7/site-packages (from boto3)\n",
      "Requirement already satisfied: botocore<1.27.0,>=1.26.10 in /usr/local/lib/python3.7/site-packages (from boto3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/local/lib/python3.7/site-packages (from botocore<1.27.0,>=1.26.10->boto3)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.7/site-packages (from botocore<1.27.0,>=1.26.10->boto3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.27.0,>=1.26.10->boto3)\n",
      "Installing collected packages: boto3\n",
      "Successfully installed boto3-1.23.10\n",
      "\n",
      "Collecting seaborn\n",
      "  Using cached https://files.pythonhosted.org/packages/10/5b/0479d7d845b5ba410ca702ffcd7f2cd95a14a4dfff1fde2637802b258b9b/seaborn-0.11.2-py3-none-any.whl\n",
      "Requirement already satisfied: numpy>=1.15 in /usr/local/lib64/python3.7/site-packages (from seaborn)\n",
      "Collecting scipy>=1.0 (from seaborn)\n",
      "  Using cached https://files.pythonhosted.org/packages/61/67/1a654b96309c991762ee9bc39c363fc618076b155fe52d295211cf2536c7/scipy-1.7.3.tar.gz\n",
      "Collecting matplotlib>=2.2 (from seaborn)\n",
      "  Using cached https://files.pythonhosted.org/packages/df/e7/0ad4aad00d6d0314aaf97526a54a34d385f898ebb7915d112431fff452ff/matplotlib-3.5.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl\n",
      "Collecting pandas>=0.23 (from seaborn)\n",
      "  Using cached https://files.pythonhosted.org/packages/99/f0/f99700ef327e51d291efdf4a6de29e685c4d198cbf8531541fc84d169e0e/pandas-1.3.5.tar.gz\n",
      "    Complete output from command python setup.py egg_info:\n",
      "    Traceback (most recent call last):\n",
      "      File \"<string>\", line 1, in <module>\n",
      "      File \"/mnt/tmp/pip-build-5g2_5hjm/pandas/setup.py\", line 650, in <module>\n",
      "        ext_modules=maybe_cythonize(extensions, compiler_directives=directives),\n",
      "      File \"/mnt/tmp/pip-build-5g2_5hjm/pandas/setup.py\", line 414, in maybe_cythonize\n",
      "        raise RuntimeError(\"Cannot cythonize without Cython installed.\")\n",
      "    RuntimeError: Cannot cythonize without Cython installed.\n",
      "    \n",
      "    ----------------------------------------\n",
      "\n",
      "Collecting pandas==0.23.2\n",
      "  Using cached https://files.pythonhosted.org/packages/ab/de/e936ebd45b3200a674e4ec5404c0d2d87b9130bb9ac5cc286fd778e0bb29/pandas-0.23.2-cp37-cp37m-manylinux1_x86_64.whl\n",
      "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib64/python3.7/site-packages (from pandas==0.23.2)\n",
      "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.7/site-packages (from pandas==0.23.2)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.7/site-packages (from pandas==0.23.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/site-packages (from python-dateutil>=2.5.0->pandas==0.23.2)\n",
      "Installing collected packages: pandas\n",
      "Successfully installed pandas-0.23.2\n",
      "\n",
      "\n",
      "Command \"python setup.py egg_info\" failed with error code 1 in /mnt/tmp/pip-build-5g2_5hjm/pandas/"
     ]
    }
   ],
   "source": [
    "sc.install_pypi_package(\"boto3\") # use boto3 to see book data in S3 bucket\n",
    "sc.install_pypi_package(\"seaborn\")\n",
    "sc.install_pypi_package(\"pandas==0.23.2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "46d587e1-d2bf-4f31-8155-0b3437e07419",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:17:59.309274Z",
     "iopub.status.busy": "2022-05-30T18:17:59.309095Z",
     "iopub.status.idle": "2022-05-30T18:18:00.105016Z",
     "shell.execute_reply": "2022-05-30T18:18:00.104330Z",
     "shell.execute_reply.started": "2022-05-30T18:17:59.309251Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "357fc86e0ae44208886d1fd69404bfc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import boto3\n",
    "\n",
    "from math import ceil\n",
    "# tqdm provides a progress bar for iteration \n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "import pyspark.sql.functions as f\n",
    "from pyspark.sql.types import (\n",
    "    StructType, StructField, StringType, DateType\n",
    ")\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import (\n",
    "    CountVectorizer, RegexTokenizer,\n",
    "    IDF, StopWordsRemover\n",
    ")\n",
    "from pyspark.ml.clustering import LDA\n",
    "#Spark NLP\n",
    "from sparknlp.annotator import *\n",
    "from sparknlp.base import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1d7ae10",
   "metadata": {},
   "source": [
    "## Retrieve data from S3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "75b8ca23-8cd0-4017-b035-ec7def9b398e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:00.106295Z",
     "iopub.status.busy": "2022-05-30T18:18:00.106105Z",
     "iopub.status.idle": "2022-05-30T18:18:05.419782Z",
     "shell.execute_reply": "2022-05-30T18:18:05.419069Z",
     "shell.execute_reply.started": "2022-05-30T18:18:00.106271Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aeed6a82bb84b6c8a4e5a258553dfcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = spark.read.csv('s3://lynette-dang-final-project-data/speech_data.csv', \n",
    "                      header=True, \n",
    "                      sep=',', \n",
    "                      escape=\"\\\"\", \n",
    "                      multiLine=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e1599631-10fb-48d9-9dd6-b28b8070caf6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:05.420999Z",
     "iopub.status.busy": "2022-05-30T18:18:05.420802Z",
     "iopub.status.idle": "2022-05-30T18:18:08.795781Z",
     "shell.execute_reply": "2022-05-30T18:18:08.795215Z",
     "shell.execute_reply.started": "2022-05-30T18:18:05.420976Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d69dc230323c4e6785063ed15d872132",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Columns: 8\n",
      "Total Rows: 719\n",
      "root\n",
      " |-- _c0: string (nullable = true)\n",
      " |-- Unnamed: 0: string (nullable = true)\n",
      " |-- Legislator name: string (nullable = true)\n",
      " |-- Legislative Effectiveness Score: string (nullable = true)\n",
      " |-- party: string (nullable = true)\n",
      " |-- Name5: string (nullable = true)\n",
      " |-- name6: string (nullable = true)\n",
      " |-- speech: string (nullable = true)"
     ]
    }
   ],
   "source": [
    "print('Total Columns: %d' % len(data.dtypes))\n",
    "print('Total Rows: %d' % data.count())\n",
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b19c433-99c0-41dd-bb5d-217aa6dc675f",
   "metadata": {},
   "source": [
    "## Proprecessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "49c249ff-d5a2-4a6d-8358-379580daf121",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:08.796859Z",
     "iopub.status.busy": "2022-05-30T18:18:08.796691Z",
     "iopub.status.idle": "2022-05-30T18:18:11.086431Z",
     "shell.execute_reply": "2022-05-30T18:18:11.080813Z",
     "shell.execute_reply.started": "2022-05-30T18:18:08.796838Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02aa9b19e58349389e72d1f123a85096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------+---------------+-------------------------------+-----+--------+--------+--------------------+--------------------+\n",
      "|_c0|Unnamed: 0|Legislator name|Legislative Effectiveness Score|party|   Name5|   name6|              speech|               words|\n",
      "+---+----------+---------------+-------------------------------+-----+--------+--------+--------------------+--------------------+\n",
      "|  0|         0|  Ralph Abraham|             0.0397605448961257|    R| abraham| abraham| mr. abraham. mr....|[, mr., abraham.,...|\n",
      "|  1|         1|     Alma Adams|             1.7837893962860107|    D|   adams|   adams| ms. adams. mr. s...|[, ms., adams., m...|\n",
      "|  2|         2|Robert Aderholt|             0.0198802724480628|    R|aderholt|aderholt| mr. aderholt. mr...|[, mr., aderholt....|\n",
      "|  3|         3|   Pete Aguilar|             0.0497006773948669|    D| aguilar| aguilar| mr. aguilar. mad...|[, mr., aguilar.,...|\n",
      "|  4|         4|     Rick Allen|             0.0397605448961257|    R|   allen|   allen| mr. allen. mr. s...|[, mr., allen., m...|\n",
      "+---+----------+---------------+-------------------------------+-----+--------+--------+--------------------+--------------------+\n",
      "only showing top 5 rows"
     ]
    }
   ],
   "source": [
    "data = data.withColumn('speech', f.lower(f.col('speech'))).withColumn('words',f.split('speech', ' '))\n",
    "data.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0718ca4d-7e29-4712-b0a0-9255c9fa2a85",
   "metadata": {},
   "source": [
    "## Build and Apply Spark NLP Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6817eaac",
   "metadata": {},
   "source": [
    "### 1. Document Assembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b4080b64-1cdf-4a3f-8123-d107c8e0732c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:11.670610Z",
     "iopub.status.busy": "2022-05-30T18:18:11.670359Z",
     "iopub.status.idle": "2022-05-30T18:18:11.960598Z",
     "shell.execute_reply": "2022-05-30T18:18:11.957099Z",
     "shell.execute_reply.started": "2022-05-30T18:18:11.670576Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccdc8f3cb7d64aaa8ec65f0b9f092eaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sparknlp.base import DocumentAssembler\n",
    "\n",
    "documentAssembler = DocumentAssembler() \\\n",
    "     .setInputCol('speech') \\\n",
    "     .setOutputCol('document')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fde79a6e",
   "metadata": {},
   "source": [
    "### 2. Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f1ba6d90-8a1e-4a32-9289-e3663457af92",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:11.961935Z",
     "iopub.status.busy": "2022-05-30T18:18:11.961718Z",
     "iopub.status.idle": "2022-05-30T18:18:12.237306Z",
     "shell.execute_reply": "2022-05-30T18:18:12.236598Z",
     "shell.execute_reply.started": "2022-05-30T18:18:11.961906Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78bc0d1e88aa4db3a99adc223f257d75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sparknlp.annotator import Tokenizer\n",
    "\n",
    "tokenizer = Tokenizer() \\\n",
    "     .setInputCols(['document']) \\\n",
    "     .setOutputCol('tokenized')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2816ccfa",
   "metadata": {},
   "source": [
    "### 3.  Normalizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9eacac26-0d6c-4f6b-a09b-21d8485d516f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:12.238868Z",
     "iopub.status.busy": "2022-05-30T18:18:12.238508Z",
     "iopub.status.idle": "2022-05-30T18:18:12.315653Z",
     "shell.execute_reply": "2022-05-30T18:18:12.314972Z",
     "shell.execute_reply.started": "2022-05-30T18:18:12.238840Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f237a562461e464aa75398ece805f9c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sparknlp.annotator import Normalizer\n",
    "\n",
    "normalizer = Normalizer() \\\n",
    "     .setInputCols(['tokenized']) \\\n",
    "     .setOutputCol('normalized') \\\n",
    "     .setLowercase(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8e6d332",
   "metadata": {},
   "source": [
    "### 4. Lemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "702945d1-5c6c-483d-9bf0-96c669aaa970",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:12.317312Z",
     "iopub.status.busy": "2022-05-30T18:18:12.316881Z",
     "iopub.status.idle": "2022-05-30T18:18:21.718120Z",
     "shell.execute_reply": "2022-05-30T18:18:21.717418Z",
     "shell.execute_reply.started": "2022-05-30T18:18:12.317274Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f26de6c5988a4112befd3b2725fe70d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lemma_antbnc download started this may take some time.\n",
      "Approximate size to download 907.6 KB\n",
      "[OK!]"
     ]
    }
   ],
   "source": [
    "from sparknlp.annotator import LemmatizerModel\n",
    "\n",
    "lemmatizer = LemmatizerModel.pretrained() \\\n",
    "     .setInputCols(['normalized']) \\\n",
    "     .setOutputCol('lemmatized')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23856a5b",
   "metadata": {},
   "source": [
    "### 5. Stopwords Remover"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "610ebe88-4b23-465b-977a-f249ff5bf799",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:21.719229Z",
     "iopub.status.busy": "2022-05-30T18:18:21.719062Z",
     "iopub.status.idle": "2022-05-30T18:18:22.485608Z",
     "shell.execute_reply": "2022-05-30T18:18:22.484951Z",
     "shell.execute_reply.started": "2022-05-30T18:18:21.719208Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf51c9c9cb7041908f58a3bad2fde712",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /var/lib/livy/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "eng_stopwords = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4c1361e2-a9f3-4d31-9d89-7435b4139078",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:22.486612Z",
     "iopub.status.busy": "2022-05-30T18:18:22.486442Z",
     "iopub.status.idle": "2022-05-30T18:18:22.754587Z",
     "shell.execute_reply": "2022-05-30T18:18:22.753910Z",
     "shell.execute_reply.started": "2022-05-30T18:18:22.486590Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "560fa1b09d5f468fbc4135bbd1ae5ec8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sparknlp.annotator import StopWordsCleaner\n",
    "\n",
    "stopwords_cleaner = StopWordsCleaner() \\\n",
    "     .setInputCols(['lemmatized']) \\\n",
    "     .setOutputCol('unigrams') \\\n",
    "     .setStopWords(eng_stopwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "959bade1",
   "metadata": {},
   "source": [
    "### 6. N-Gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b7fb97ee-d289-4cad-b95e-d6efa105db1b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:22.756038Z",
     "iopub.status.busy": "2022-05-30T18:18:22.755617Z",
     "iopub.status.idle": "2022-05-30T18:18:22.815298Z",
     "shell.execute_reply": "2022-05-30T18:18:22.814736Z",
     "shell.execute_reply.started": "2022-05-30T18:18:22.755999Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d00868312310411ca6d89854c6b9fd5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sparknlp.annotator import NGramGenerator\n",
    "\n",
    "ngrammer = NGramGenerator() \\\n",
    "    .setInputCols(['lemmatized']) \\\n",
    "    .setOutputCol('ngrams') \\\n",
    "    .setN(3) \\\n",
    "    .setEnableCumulative(True) \\\n",
    "    .setDelimiter('_')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0cf5ce08-f091-47b9-ba3c-989f5ee918ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:22.816419Z",
     "iopub.status.busy": "2022-05-30T18:18:22.816213Z",
     "iopub.status.idle": "2022-05-30T18:18:32.130153Z",
     "shell.execute_reply": "2022-05-30T18:18:32.129480Z",
     "shell.execute_reply.started": "2022-05-30T18:18:22.816393Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f83f66bfdc63456784d5aa2656545101",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos_anc download started this may take some time.\n",
      "Approximate size to download 3.9 MB\n",
      "[OK!]"
     ]
    }
   ],
   "source": [
    "from sparknlp.annotator import PerceptronModel\n",
    "\n",
    "pos_tagger = PerceptronModel.pretrained('pos_anc') \\\n",
    "    .setInputCols(['document', 'lemmatized']) \\\n",
    "    .setOutputCol('pos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "25b6765a-d31b-444d-83df-9a540770e1c9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:32.131599Z",
     "iopub.status.busy": "2022-05-30T18:18:32.131338Z",
     "iopub.status.idle": "2022-05-30T18:18:32.188665Z",
     "shell.execute_reply": "2022-05-30T18:18:32.187997Z",
     "shell.execute_reply.started": "2022-05-30T18:18:32.131563Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "083cb63d973047b5947d0f28bcfc54a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sparknlp.base import Finisher\n",
    "\n",
    "finisher = Finisher() \\\n",
    "     .setInputCols(['unigrams', 'ngrams', 'pos']) \\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3b7dc7f2-291f-4827-aefb-9cb1273fd8a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:18:32.189717Z",
     "iopub.status.busy": "2022-05-30T18:18:32.189553Z",
     "iopub.status.idle": "2022-05-30T18:18:32.252763Z",
     "shell.execute_reply": "2022-05-30T18:18:32.251933Z",
     "shell.execute_reply.started": "2022-05-30T18:18:32.189694Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7f1290f25544daab84c59780f8b3044",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "\n",
    "pipeline = Pipeline() \\\n",
    "     .setStages([documentAssembler,                  \n",
    "                 tokenizer,\n",
    "                 normalizer,                  \n",
    "                 lemmatizer,                  \n",
    "                 stopwords_cleaner, \n",
    "                 pos_tagger,\n",
    "                 ngrammer,  \n",
    "                 finisher])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2ce5685",
   "metadata": {},
   "source": [
    "### 7. Use this pipeline to transform the speeches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "17c8e3b8-291e-47e4-8316-853ad8519841",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:19:19.625397Z",
     "iopub.status.busy": "2022-05-30T18:19:19.625151Z",
     "iopub.status.idle": "2022-05-30T18:19:20.897430Z",
     "shell.execute_reply": "2022-05-30T18:19:20.896799Z",
     "shell.execute_reply.started": "2022-05-30T18:19:19.625371Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d12a16e1ca44daeb6e6548b031158e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pyspark.sql.functions as F\n",
    "\n",
    "speech =  data.select('speech').filter(F.col('speech').isNotNull())\n",
    "processed_speech = pipeline.fit(speech).transform(speech)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "6039b113-cd96-4b22-9fbf-1d140ff76a67",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:19:29.108351Z",
     "iopub.status.busy": "2022-05-30T18:19:29.108092Z",
     "iopub.status.idle": "2022-05-30T18:19:34.399991Z",
     "shell.execute_reply": "2022-05-30T18:19:34.399405Z",
     "shell.execute_reply.started": "2022-05-30T18:19:29.108324Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab128aaf95074ac6b79ce823ace91462",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "|              speech|   finished_unigrams|     finished_ngrams|        finished_pos|\n",
      "+--------------------+--------------------+--------------------+--------------------+\n",
      "| mr. abraham. mr....|[mr, abraham, mr,...|[mr, abraham, mr,...|[NNP, NN, NNP, NN...|\n",
      "| ms. adams. mr. s...|[ms, adam, mr, sp...|[ms, adam, mr, sp...|[NN, NN, NNP, NN,...|\n",
      "| mr. aderholt. mr...|[mr, aderholt, mr...|[mr, aderholt, mr...|[NNP, NN, NNP, NN...|\n",
      "| mr. aguilar. mad...|[mr, aguilar, mad...|[mr, aguilar, mad...|[NNP, NN, NN, NN,...|\n",
      "| mr. allen. mr. s...|[mr, allen, mr, s...|[mr, allen, mr, s...|[NNP, NN, NNP, NN...|\n",
      "+--------------------+--------------------+--------------------+--------------------+"
     ]
    }
   ],
   "source": [
    "processed_speech.limit(5).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12c5fe1c",
   "metadata": {},
   "source": [
    "## Build LDA model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a0d323a",
   "metadata": {},
   "source": [
    "### 1. Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d72d4a2c-a2ac-43a8-bf88-2aac43ee2b1d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:19:34.401190Z",
     "iopub.status.busy": "2022-05-30T18:19:34.401005Z",
     "iopub.status.idle": "2022-05-30T18:19:51.821313Z",
     "shell.execute_reply": "2022-05-30T18:19:51.820678Z",
     "shell.execute_reply.started": "2022-05-30T18:19:34.401167Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6e86466320e4736a0242c07d0e52018",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.feature import CountVectorizer\n",
    "\n",
    "tfizer = CountVectorizer(inputCol='finished_ngrams', outputCol='tf')\n",
    "tf_model = tfizer.fit(processed_speech)\n",
    "tf_result = tf_model.transform(processed_speech)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceecdf32",
   "metadata": {},
   "source": [
    "### 2. Tf/Idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e35ea52f-c71e-44c7-b5a3-6256cf0f6638",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:19:51.823167Z",
     "iopub.status.busy": "2022-05-30T18:19:51.822897Z",
     "iopub.status.idle": "2022-05-30T18:20:05.132020Z",
     "shell.execute_reply": "2022-05-30T18:20:05.131461Z",
     "shell.execute_reply.started": "2022-05-30T18:19:51.823116Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb197c211381466e9e9bc03878510390",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.feature import IDF\n",
    "\n",
    "idfizer = IDF(inputCol='tf', outputCol='tf_idf')\n",
    "idf_model = idfizer.fit(tf_result)\n",
    "tfidf_result = idf_model.transform(tf_result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c17c1c4c",
   "metadata": {},
   "source": [
    "### 3. Use tf/idf to build LDA model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "fb710f9b-aed3-4a63-9c06-c6c2102af1f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:20:05.133664Z",
     "iopub.status.busy": "2022-05-30T18:20:05.133414Z",
     "iopub.status.idle": "2022-05-30T18:24:04.387672Z",
     "shell.execute_reply": "2022-05-30T18:24:04.386912Z",
     "shell.execute_reply.started": "2022-05-30T18:20:05.133630Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee1208c911d549b4a78f436f4d8775c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.clustering import LDA\n",
    "\n",
    "num_topics = 50\n",
    "max_iter = 10\n",
    "\n",
    "lda = LDA(k=num_topics, maxIter=max_iter, featuresCol='tf_idf')\n",
    "lda_model = lda.fit(tfidf_result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b210366",
   "metadata": {},
   "source": [
    "## Get the top 50 issues and related keywords from all speeches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2d585e98-fadb-4e1e-a61f-069bb05a183b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:24:04.389187Z",
     "iopub.status.busy": "2022-05-30T18:24:04.388786Z",
     "iopub.status.idle": "2022-05-30T18:24:06.681244Z",
     "shell.execute_reply": "2022-05-30T18:24:06.680451Z",
     "shell.execute_reply.started": "2022-05-30T18:24:04.389147Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be416e40bdee4660b56080b2b011828c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vocab = tf_model.vocabulary\n",
    "from pyspark.sql import types as T\n",
    "\n",
    "\n",
    "def get_words(token_list):\n",
    "     return [vocab[token_id] for token_id in token_list]\n",
    "       \n",
    "udf_to_words = F.udf(get_words, T.ArrayType(T.StringType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d8d4adef-232d-4f03-863b-f9ddb5aabc0d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:24:06.682905Z",
     "iopub.status.busy": "2022-05-30T18:24:06.682443Z",
     "iopub.status.idle": "2022-05-30T18:24:20.065895Z",
     "shell.execute_reply": "2022-05-30T18:24:20.065274Z",
     "shell.execute_reply.started": "2022-05-30T18:24:06.682869Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1f9d123c78d41e6a1b4e1a8c5e2b2bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_top_words = 10\n",
    "\n",
    "topics = lda_model.describeTopics(num_top_words).withColumn('topicWords', udf_to_words(F.col('termIndices')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bdb7f983-dde5-44f1-a7cf-56ef37a32ee3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:24:20.067231Z",
     "iopub.status.busy": "2022-05-30T18:24:20.066971Z",
     "iopub.status.idle": "2022-05-30T18:24:20.832815Z",
     "shell.execute_reply": "2022-05-30T18:24:20.832098Z",
     "shell.execute_reply.started": "2022-05-30T18:24:20.067194Z"
    },
    "scrolled": false,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e85aceb278de4bf5b932d5ec4abd2dc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|topic|topicWords                                                                                                                                                                              |\n",
      "+-----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|0    |[section, section_of, department, information, health, treasury, the_department, act, act_of, bankruptcy]                                                                               |\n",
      "|1    |[unanimous_consent, unanimous, pennsylvania, consent, dialysis, america_small, america_small_business, organ, paycheck_protection, i_ask_unanimous]                                     |\n",
      "|2    |[child_abuse, beyer, ks, davids, raskin, ma, pallone, tx, clark_ma, davids_ks]                                                                                                          |\n",
      "|3    |[giuliani, insulin, ukrainian, arizonans, a_word_of, delight, the_lonely, the_hold, zelensky, i_be_delight]                                                                             |\n",
      "|4    |[bankruptcy, bankruptcy_judgeship, judgeship, the_bankruptcy, bankruptcy_judge, bankruptcy_judgeship_act, usc_note, the_bankruptcy_judgeship, a_bankruptcy_judge, of_usc_note]          |\n",
      "|5    |[vicki, brain_cancer, brain, speed, city_of_brockton, indiana, you, operation_warp, distribution_of_the, brockton]                                                                      |\n",
      "|6    |[distribute_wind, song, this_song, james_weldon, weldon, jacksonville, wind_power, of_jacksonville, wind, the_district_i]                                                               |\n",
      "|7    |[classmate, i_classmate, long_gray_line, devos, state_military_academy, the_long_gray, the_frustrate, gray, the_house_should, platoon]                                                  |\n",
      "|8    |[will_join, will_join_i, caribbean_basin, caribbean, the_caribbean, basin, you_will_join, basin_trade_promotion, trade_promotion, the_caribbean_basin]                                  |\n",
      "|9    |[god, i_dad, lisa, bible, abortion, she, the_bible, go, dad, if]                                                                                                                        |\n",
      "|10   |[bankruptcy, bankruptcy_judge, bankruptcy_judgeship, judgeship, bankruptcy_judgeship_act, the_bankruptcy_judgeship, the_bankruptcy, judgeship_act, a_bankruptcy_judge, judgeship_act_of]|\n",
      "|11   |[farm, nebraska, farm_bureau, the_great_lake, bureau, great_lake, lake, steve, as_president, nebraska_farm_bureau]                                                                      |\n",
      "|12   |[dejoy, of_the_vaccine, dan, child_abuse, vicki, vaccine_will, after_month_of, after_month, on_the_postal, chaos_be]                                                                    |\n",
      "|13   |[pension, bankruptcy, bankruptcy_judgeship, session, county, repay, mayor, the_butch_lewis, butch, loan]                                                                                |\n",
      "|14   |[the_caribbean, trade, caribbean_basin, have_be_tough, caribbean, outofnetwork, be_continue, basin, the_caribbean_basin, caribbean_basin_trade]                                         |\n",
      "|15   |[neil, armstrong, apollo, suspend, the_question_be, the_question, neil_armstrong, motion, question_be, native_hawaiian]                                                                 |\n",
      "|16   |[belarus, ukraine, sanction, of_belarus, election, democracy, of_the_senate, to_meet_during, protester, session_of]                                                                     |\n",
      "|17   |[giuliani, ukrainian, ukraine, zelensky, the_ukrainian, ambassador, ukraines, belarus, volker, in_ukraine]                                                                              |\n",
      "|18   |[ambassador, ukrainian, giuliani, the_hold, hawaiian, native_hawaiian, carter, in_ukraine, jimmy_carter, caribbean]                                                                     |\n",
      "|19   |[asian, antiasian, asian_american, of_racism, of_antiasian, bigotry, condemn_all, scapegoat, antiasian_sentiment, covid_we]                                                             |\n",
      "|20   |[giuliani, wait_period, individual_with, with_disability, the_arena, in_the_arena, consent_that_the, harley, beneficial, individual_with_disability]                                    |\n",
      "|21   |[business, insurance, billion, small_business, ppp, billion_for, small, loan, affordable, they_application]                                                                             |\n",
      "|22   |[you, we, he, they, she, military, that, do, american, child]                                                                                                                           |\n",
      "|23   |[whub, tennessee, tennessee_and, upper, the_upper_cumberland, the_upper, radio_station, cumberland, upper_cumberland, radio_station_in]                                                 |\n",
      "|24   |[puerto, puerto_rico, rico, bankruptcy, section, bankruptcy_judge, judgeship, of_puerto_rico, of_puerto, act_of_usc]                                                                    |\n",
      "|25   |[scout, song, to_commemorate_the, to_commemorate, commemorate, commemorate_the, weldon, arizonans, james_weldon, rise_to_commemorate]                                                   |\n",
      "|26   |[child_abuse, hotline, neglect, maine, and_neglect, national_child_abuse, abuse_and_neglect, child_abuse_and, national_child, child_abuse_hotline]                                      |\n",
      "|27   |[child_abuse, and_neglect, neglect, national_child, national_child_abuse, abuse_and_neglect, child_abuse_hotline, child_abuse_and, maine, maltreatment]                                 |\n",
      "|28   |[beyer, ks, garcia, davids_ks, the_math, giuliani, math, cicilline, davids, raskin]                                                                                                     |\n",
      "|29   |[shelby_county, shelby, county, suzette, the_shelby, judge, session, general_session, the_shelby_county, work_for_the]                                                                  |\n",
      "|30   |[plan, latino, carter, billion, she, they, health, bill, hr, officer]                                                                                                                   |\n",
      "|31   |[division_f, ownership, tom_casperson, beneficial_ownership_information, casperson, beneficial_ownership, to_shave, beneficial, bullock, a_state_senator]                               |\n",
      "|32   |[cub, wgntv, the_cub, wgn, cub_baseball, baseball, cub_fan, game, ernie, wrigley]                                                                                                       |\n",
      "|33   |[bankruptcy, judgeship, bankruptcy_judgeship, the_bankruptcy, tom, bankruptcy_judgeship_act, bankruptcy_judge, usc_note, the_bankruptcy_judgeship, judgeship_act]                       |\n",
      "|34   |[useful_for, appropriation_process, useful, the_math, military_spend, these_appropriation, let_do, richard, appropriation_bill, conservative]                                           |\n",
      "|35   |[indiana, of_indiana, the_fifth_district, the_fifth, hoosiers, fifth_district, hoosier, friend_representative, biological, indianapolis]                                                |\n",
      "|36   |[cannabis, voucher_for_payment, payment_of, payment_of_money, voucher_for, for_payment, of_money, voucher, for_payment_of, personnel_action]                                            |\n",
      "|37   |[military_spend, conservative, syrian, syrian_civil_war, the_syrian, the_syrian_civil, syrian_civil, more_military_spend, when_congress, that_we_military]                              |\n",
      "|38   |[dan, reporter, dan_to, dan_have, report, honest_report, honest, set_a, he, connecticut]                                                                                                |\n",
      "|39   |[insulin, india, overwhelm, dr_v, hospital, we_hospital, be_overwhelm, sick_in_texas, medicaid, price_and]                                                                              |\n",
      "|40   |[classmate, beyer, long_gray, cadet, davids, ks, tx, no_on_rollcall, jeffries, raskin]                                                                                                  |\n",
      "|41   |[giuliani, ukrainian, the_hold, cycle, volker, in_ukraine, omb, boyd, boyd_cycle, zelensky]                                                                                             |\n",
      "|42   |[cub, wgntv, baseball, the_cub, belarus, wgn, cub_baseball, game_on, nebraska, on_wgn]                                                                                                  |\n",
      "|43   |[bankruptcy, bankruptcy_judge, bankruptcy_judgeship, judgeship, usc_note, you, judgeship_act, district_of, the_bankruptcy, of_usc_note]                                                 |\n",
      "|44   |[question, the_resolution_the, question_be, the_previous_question, previous_question, the_resolution, the_question_be, the_question, the_previous, on_the_resolution]                   |\n",
      "|45   |[giuliani, johnnys, of_covid, testify, free_of_covid, covid_unless, mr_espaillat_we, be_free_of, covid_unless_the, be_page_h]                                                           |\n",
      "|46   |[boyd, she, bike, nicole, boyd_cycle, cycle, maia, boyd_and_nicole, boyd_and, and_nicole]                                                                                               |\n",
      "|47   |[student, student_debt, they_student, latino, substantially, debt, year_institution, substantially_on, institution, substantially_on_the]                                               |\n",
      "|48   |[bob, ranch, he, i_dad, wonderful_mentor_to, wonderful_mentor, a_wonderful_mentor, giuliani, be_a_wonderful, a_wonderful]                                                               |\n",
      "|49   |[beyer, jeffries, on_rollcall, raskin, rollcall, tx, clark_ma, ma, rollcall_no, on_rollcall_no]                                                                                         |\n",
      "+-----+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+"
     ]
    }
   ],
   "source": [
    "topics.select('topic', 'topicWords').show(50, False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7874ae39-7ecc-429d-8591-717e75c2a67c",
   "metadata": {},
   "source": [
    "## Manual Issue Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "630ef18c-25a6-4d73-9b65-f527410ac81c",
   "metadata": {},
   "source": [
    "From the topics and related words from above, I chose and assembled ten issues and related key words for the construction of issue-oriented cross-partisan social networks.\n",
    "\n",
    "Because I only have limited amount of congressional speech (meaning that the length of each legislator's speech is not very long), I decided to only construct 10 issue-based networks instead of having more issues. The way I construct it is to record the frequency of the key words related to each issue, if any of the key words is mentioned in a legislator's speech, map this legislator to the network, and write each issue-based network to s3.\n",
    "\n",
    "The rest of the analysis will take place locally.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "bd469c01-820a-488e-95af-c032dda4e6ef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:24:21.092766Z",
     "iopub.status.busy": "2022-05-30T18:24:21.092589Z",
     "iopub.status.idle": "2022-05-30T18:24:21.143935Z",
     "shell.execute_reply": "2022-05-30T18:24:21.143380Z",
     "shell.execute_reply.started": "2022-05-30T18:24:21.092744Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "412de6b814554ae492a0e68565326552",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "covid = [\"covid\", \"pandemic\", \"epidemic\", \"vaccine\", \"quarantine\", \"mask\"]\n",
    "bankruptcy= [\"bankruptcy\", \"judge\", \"judgeship\"]\n",
    "child = [\"child\", \"abuse\", \"children\", \"neglect\", \"hotline\"]\n",
    "rights = [\"latino\", \"asian\", \"hate\", \"lgbt\", \"lgbtq\"]\n",
    "ukraine = [\"putin\", \"ukraine\", \"ukranian\", \"ambassador\", \"military\", \"russia\", \"belarus\", \"zelensky\", \"war\"]\n",
    "business = [\"loan\", \"small\", \"affordable\", \"application\", \"trade\", \"application\"]\n",
    "syria = [\"military\", \"civil\", \"war\", \"syria\"]\n",
    "health_care = [\"medicare\", \"health\", \"insurance\", \"hospital\", \"insulin\"]\n",
    "abortion = [\"bible\", \"woman\", \"women\", \"god\", \"abortion\"]\n",
    "education = [\"student\", \"debt\", \"insitution\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "e2d3e061-cb2b-4a45-aed2-d55a27fa19b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:24:21.144994Z",
     "iopub.status.busy": "2022-05-30T18:24:21.144827Z",
     "iopub.status.idle": "2022-05-30T18:24:21.200230Z",
     "shell.execute_reply": "2022-05-30T18:24:21.199431Z",
     "shell.execute_reply.started": "2022-05-30T18:24:21.144972Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2786a2b617a4d1aa0e456f70d305e7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from functools import partial\n",
    "import re\n",
    "\n",
    "def count_values(inp,map_list=None):  \n",
    "    count = 0\n",
    "    for pattern in map_list:\n",
    "        if re.findall(pattern,inp):\n",
    "            count += 1\n",
    "    return count\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "7f483b73-c484-4eb2-b183-1e3eeb76d40d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:15.749688Z",
     "iopub.status.busy": "2022-05-30T18:25:15.749413Z",
     "iopub.status.idle": "2022-05-30T18:25:16.029878Z",
     "shell.execute_reply": "2022-05-30T18:25:16.029096Z",
     "shell.execute_reply.started": "2022-05-30T18:25:15.749661Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24eaccb1813947bbbc31598f3e1c8f8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=covid))\n",
    "new_data = data.withColumn(\"count_covid\", count_values_udf(F.col('speech')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f4e86319-29d1-427c-87d3-8ceb8492bf06",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:21.976014Z",
     "iopub.status.busy": "2022-05-30T18:25:21.975773Z",
     "iopub.status.idle": "2022-05-30T18:25:22.235447Z",
     "shell.execute_reply": "2022-05-30T18:25:22.234791Z",
     "shell.execute_reply.started": "2022-05-30T18:25:21.975989Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "083782fe33614d1c9db9032d7d649aba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=bankruptcy))\n",
    "new_data_2 = new_data.withColumn(\"count_bankruptcy\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "6f00d041-7798-4102-b6be-c495e4b3063c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:27.076522Z",
     "iopub.status.busy": "2022-05-30T18:25:27.076269Z",
     "iopub.status.idle": "2022-05-30T18:25:27.330801Z",
     "shell.execute_reply": "2022-05-30T18:25:27.330185Z",
     "shell.execute_reply.started": "2022-05-30T18:25:27.076496Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1ef643c1fd84a3e99d460b7c06d2825",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=child))\n",
    "new_data_3 = new_data_2.withColumn(\"count_child\", count_values_udf(F.col('speech')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "c99b848b-574e-4b69-8beb-cef21d5e0e4a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:29.874497Z",
     "iopub.status.busy": "2022-05-30T18:25:29.874261Z",
     "iopub.status.idle": "2022-05-30T18:25:30.136052Z",
     "shell.execute_reply": "2022-05-30T18:25:30.135385Z",
     "shell.execute_reply.started": "2022-05-30T18:25:29.874470Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb46272d79be41cd9809e45b53b610bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=rights))\n",
    "new_data_4 = new_data_3.withColumn(\"count_rights\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "1efab3fa-26d9-4c40-ae83-4eb3d89c349c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:32.670136Z",
     "iopub.status.busy": "2022-05-30T18:25:32.669914Z",
     "iopub.status.idle": "2022-05-30T18:25:32.930033Z",
     "shell.execute_reply": "2022-05-30T18:25:32.929343Z",
     "shell.execute_reply.started": "2022-05-30T18:25:32.670112Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78bf9ea448784cf39c0996a816f025a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=ukraine))\n",
    "new_data_5 = new_data_4.withColumn(\"count_ukraine\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "4e0a215c-4e8b-4972-8607-1eb51b8ce9f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:35.870002Z",
     "iopub.status.busy": "2022-05-30T18:25:35.869776Z",
     "iopub.status.idle": "2022-05-30T18:25:36.124847Z",
     "shell.execute_reply": "2022-05-30T18:25:36.124119Z",
     "shell.execute_reply.started": "2022-05-30T18:25:35.869978Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93290d7316764d12b5b2b583764654f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=business))\n",
    "new_data_6 = new_data_5.withColumn(\"count_business\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "4a4bafe9-2af2-4433-80fd-ab622099b09e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:54.805004Z",
     "iopub.status.busy": "2022-05-30T18:25:54.804778Z",
     "iopub.status.idle": "2022-05-30T18:25:55.062396Z",
     "shell.execute_reply": "2022-05-30T18:25:55.061839Z",
     "shell.execute_reply.started": "2022-05-30T18:25:54.804980Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ec55b9e5fdb41ab8f3f9887585c048c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=syria))\n",
    "new_data_7 = new_data_6.withColumn(\"count_syria\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5f7bc9c1-2dc0-4a88-b7ef-c6a17d186597",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:25:58.652842Z",
     "iopub.status.busy": "2022-05-30T18:25:58.652613Z",
     "iopub.status.idle": "2022-05-30T18:25:58.915465Z",
     "shell.execute_reply": "2022-05-30T18:25:58.914638Z",
     "shell.execute_reply.started": "2022-05-30T18:25:58.652818Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a172069d3548418ab3db070831bcbefe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=health_care))\n",
    "new_data_8 = new_data_7.withColumn(\"count_health_care\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "f8364229-9690-4d71-8a6e-acb4a7cbbcb2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:26:01.909171Z",
     "iopub.status.busy": "2022-05-30T18:26:01.908944Z",
     "iopub.status.idle": "2022-05-30T18:26:02.172947Z",
     "shell.execute_reply": "2022-05-30T18:26:02.172311Z",
     "shell.execute_reply.started": "2022-05-30T18:26:01.909144Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50585f86fe38491b9f1e5c9053ecbcc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=abortion))\n",
    "new_data_9 = new_data_8.withColumn(\"count_abortion\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "3d2047b4-9995-470d-97d8-1c1c22323248",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:26:04.982624Z",
     "iopub.status.busy": "2022-05-30T18:26:04.982403Z",
     "iopub.status.idle": "2022-05-30T18:26:05.240944Z",
     "shell.execute_reply": "2022-05-30T18:26:05.240344Z",
     "shell.execute_reply.started": "2022-05-30T18:26:04.982598Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1266ba55b8647aea4c99693709b1fb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_values_udf = F.udf(partial(count_values,map_list=education))\n",
    "final_data = new_data_9.withColumn(\"count_education\", count_values_udf(F.col('speech'))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "d5f26ef4-b46a-4889-8d62-14415ef96199",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:26:07.461984Z",
     "iopub.status.busy": "2022-05-30T18:26:07.461750Z",
     "iopub.status.idle": "2022-05-30T18:26:26.777018Z",
     "shell.execute_reply": "2022-05-30T18:26:26.776339Z",
     "shell.execute_reply.started": "2022-05-30T18:26:07.461958Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35a419f06f8a49389611ce37da14ac72",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------+----------------+-------------------------------+-----+---------+---------+--------------------+--------------------+-----------+----------------+-----------+------------+-------------+--------------+-----------+-----------------+--------------+---------------+\n",
      "|_c0|Unnamed: 0| Legislator name|Legislative Effectiveness Score|party|    Name5|    name6|              speech|               words|count_covid|count_bankruptcy|count_child|count_rights|count_ukraine|count_business|count_syria|count_health_care|count_abortion|count_education|\n",
      "+---+----------+----------------+-------------------------------+-----+---------+---------+--------------------+--------------------+-----------+----------------+-----------+------------+-------------+--------------+-----------+-----------------+--------------+---------------+\n",
      "|  0|         0|   Ralph Abraham|             0.0397605448961257|    R|  abraham|  abraham| mr. abraham. mr....|[, mr., abraham.,...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              0|\n",
      "|  1|         1|      Alma Adams|             1.7837893962860107|    D|    adams|    adams| ms. adams. mr. s...|[, ms., adams., m...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              0|\n",
      "|  2|         2| Robert Aderholt|             0.0198802724480628|    R| aderholt| aderholt| mr. aderholt. mr...|[, mr., aderholt....|          2|               0|          0|           0|            0|             0|          0|                1|             0|              0|\n",
      "|  3|         3|    Pete Aguilar|             0.0497006773948669|    D|  aguilar|  aguilar| mr. aguilar. mad...|[, mr., aguilar.,...|          1|               0|          1|           0|            0|             2|          0|                0|             1|              0|\n",
      "|  4|         4|      Rick Allen|             0.0397605448961257|    R|    allen|    allen| mr. allen. mr. s...|[, mr., allen., m...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              0|\n",
      "|  5|         5|    Colin Allred|             0.1708884835243225|    D|   allred|   allred| mr. allred. mr. ...|[, mr., allred., ...|          0|               1|          0|           0|            0|             0|          0|                1|             0|              1|\n",
      "|  6|         6|    Justin Amash|             0.1789224445819854|    R|    amash|    amash| mr. amash. madam...|[, mr., amash., m...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              0|\n",
      "|  7|         7|     Mark Amodei|             0.3683782815933227|    R|   amodei|   amodei| mr. amodei. mr. ...|[, mr., amodei., ...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              0|\n",
      "|  8|         8| Kelly Armstrong|             0.0497006773948669|    R|armstrong|armstrong| mr. armstrong. m...|[, mr., armstrong...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              0|\n",
      "|  9|         9| Jodey Arrington|             0.1988027095794677|    R|arrington|arrington| mr. arrington. m...|[, mr., arrington...|          0|               0|          0|           0|            0|             1|          0|                1|             2|              0|\n",
      "| 10|        10|      Cindy Axne|              1.337895750999451|    D|     axne|     axne| mrs. axne. mr. s...|[, mrs., axne., m...|          0|               0|          0|           0|            1|             0|          1|                1|             1|              0|\n",
      "| 11|        11|     Brian Babin|             0.5621753931045532|    R|    babin|    babin| mr. babin. mr. s...|[, mr., babin., m...|          0|               0|          0|           0|            1|             1|          1|                0|             2|              0|\n",
      "| 12|        12|       Don Bacon|             0.8577378392219543|    R|    bacon|    bacon| mr. bacon. madam...|[, mr., bacon., m...|          0|               0|          2|           0|            1|             0|          1|                0|             0|              0|\n",
      "| 13|        13|     James Baird|             0.0397605448961257|    R|    baird|    baird| mr. baird. mr. s...|[, mr., baird., m...|          0|               0|          0|           0|            0|             0|          0|                0|             0|              1|\n",
      "| 14|        14|  Troy Balderson|             0.0695809498429298|    R|balderson|balderson| mr. balderson. m...|[, mr., balderson...|          0|               0|          0|           0|            0|             1|          0|                0|             0|              1|\n",
      "| 15|        15|       Jim Banks|             0.5771211385726929|    R|    banks|    banks| mr. banks. mr. s...|[, mr., banks., m...|          0|               0|          0|           0|            2|             0|          2|                0|             1|              0|\n",
      "| 16|        16|       Andy Barr|             2.2088539600372314|    R|     barr|     barr| mr. barr. madam ...|[, mr., barr., ma...|          2|               0|          0|           0|            0|             4|          0|                0|             0|              1|\n",
      "| 17|        17|Nanette Barragan|             0.8301119804382324|    D| barragan| barragan| ms. barragan. mr...|[, ms., barragan....|          1|               0|          0|           0|            0|             0|          0|                1|             1|              0|\n",
      "| 18|        18|      Karen Bass|             2.2914934158325195|    D|     bass|     bass| ms. bass. mr. sp...|[, ms., bass., mr...|          0|               0|          0|           0|            1|             0|          1|                0|             1|              0|\n",
      "| 19|        19|    Joyce Beatty|             1.2930370569229126|    D|   beatty|   beatty| mrs. beatty. mad...|[, mrs., beatty.,...|          2|               0|          0|           0|            0|             1|          0|                0|             1|              0|\n",
      "+---+----------+----------------+-------------------------------+-----+---------+---------+--------------------+--------------------+-----------+----------------+-----------+------------+-------------+--------------+-----------+-----------------+--------------+---------------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "final_data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "e61e3587-429a-48a1-b6cd-4eab7a71ce58",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:40:58.815449Z",
     "iopub.status.busy": "2022-05-30T18:40:58.815215Z",
     "iopub.status.idle": "2022-05-30T18:41:00.091887Z",
     "shell.execute_reply": "2022-05-30T18:41:00.091312Z",
     "shell.execute_reply.started": "2022-05-30T18:40:58.815424Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b49f913ff75f4893880cea095751190c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------+\n",
      "| Legislator name|\n",
      "+----------------+\n",
      "| Robert Aderholt|\n",
      "|    Pete Aguilar|\n",
      "|       Andy Barr|\n",
      "|Nanette Barragan|\n",
      "|    Joyce Beatty|\n",
      "|        Ami Bera|\n",
      "|      Dan Bishop|\n",
      "|      Rob Bishop|\n",
      "|  Sanford Bishop|\n",
      "|       Mike Bost|\n",
      "|       Mo Brooks|\n",
      "|    Susan Brooks|\n",
      "|   Vern Buchanan|\n",
      "| Michael Burgess|\n",
      "|  Salud Carbajal|\n",
      "|   Tony Cardenas|\n",
      "|    Andre Carson|\n",
      "| Matt Cartwright|\n",
      "|         Ed Case|\n",
      "|   James Clyburn|\n",
      "+----------------+\n",
      "only showing top 20 rows"
     ]
    }
   ],
   "source": [
    "covid_network =final_data.select('Legislator name').where(final_data['count_covid'] > 0) \n",
    "covid_network.show()\n",
    "covid_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/covid_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "cca87bbc-d943-4794-8e4a-36e607b11828",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:00.093064Z",
     "iopub.status.busy": "2022-05-30T18:41:00.092890Z",
     "iopub.status.idle": "2022-05-30T18:41:00.855544Z",
     "shell.execute_reply": "2022-05-30T18:41:00.854854Z",
     "shell.execute_reply.started": "2022-05-30T18:41:00.093041Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "837fa7ff619a4de682c11bb5e4394e4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bankruptcy_network =final_data.select('Legislator name').where(final_data['count_bankruptcy'] > 0) \n",
    "bankruptcy_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/bankruptcy_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "b0fe1dbb-9a67-4021-93df-bc3c5ff3ac6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:00.856961Z",
     "iopub.status.busy": "2022-05-30T18:41:00.856792Z",
     "iopub.status.idle": "2022-05-30T18:41:01.624424Z",
     "shell.execute_reply": "2022-05-30T18:41:01.623604Z",
     "shell.execute_reply.started": "2022-05-30T18:41:00.856939Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "403214c22cf44992bf42502b5ce8e983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "child_network =final_data.select('Legislator name').where(final_data['count_child'] > 0) \n",
    "child_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/child_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "3e9bf4ec-39e3-431b-800c-9f4c037a9500",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:01.625608Z",
     "iopub.status.busy": "2022-05-30T18:41:01.625445Z",
     "iopub.status.idle": "2022-05-30T18:41:02.415036Z",
     "shell.execute_reply": "2022-05-30T18:41:02.414282Z",
     "shell.execute_reply.started": "2022-05-30T18:41:01.625586Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed9928354f624aa39311f2e2e7c42f70",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rights_network =final_data.select('Legislator name').where(final_data['count_rights'] > 0) \n",
    "rights_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/rights_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "fe82ced6-ea3c-4d44-8f0c-c61d013544e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:02.428010Z",
     "iopub.status.busy": "2022-05-30T18:41:02.416111Z",
     "iopub.status.idle": "2022-05-30T18:41:03.382973Z",
     "shell.execute_reply": "2022-05-30T18:41:03.354372Z",
     "shell.execute_reply.started": "2022-05-30T18:41:02.427964Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f415b9e3787d41d08b05b08f352e4c81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ukraine_network =final_data.select('Legislator name').where(final_data['count_ukraine'] > 0) \n",
    "ukraine_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/ukraine_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "7d7bd844-352d-4c34-a070-e5caae365397",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:03.383860Z",
     "iopub.status.busy": "2022-05-30T18:41:03.383683Z",
     "iopub.status.idle": "2022-05-30T18:41:04.171339Z",
     "shell.execute_reply": "2022-05-30T18:41:04.170666Z",
     "shell.execute_reply.started": "2022-05-30T18:41:03.383837Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "024b43cc7039420b858c3a451298cb13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "business_network =final_data.select('Legislator name').where(final_data['count_business'] > 0) \n",
    "business_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/business_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "e3c25f79-eb92-4f41-848b-e35a4934c21e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:04.172690Z",
     "iopub.status.busy": "2022-05-30T18:41:04.172441Z",
     "iopub.status.idle": "2022-05-30T18:41:05.073296Z",
     "shell.execute_reply": "2022-05-30T18:41:05.072677Z",
     "shell.execute_reply.started": "2022-05-30T18:41:04.172654Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e3fcd4ad2c946298e1cecade77be8a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "syria_network =final_data.select('Legislator name').where(final_data['count_syria'] > 0) \n",
    "syria_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/syria_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "249d2f3c-30f3-429c-9947-96cec8fc0a7d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:05.074820Z",
     "iopub.status.busy": "2022-05-30T18:41:05.074646Z",
     "iopub.status.idle": "2022-05-30T18:41:05.860762Z",
     "shell.execute_reply": "2022-05-30T18:41:05.860054Z",
     "shell.execute_reply.started": "2022-05-30T18:41:05.074798Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "677850e97c0e41f29507301f25e1f992",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "health_care_network =final_data.select('Legislator name').where(final_data['count_health_care'] > 0) \n",
    "health_care_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/health_care_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "e94a2591-6105-4054-9f73-d8f0f48924cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:05.862269Z",
     "iopub.status.busy": "2022-05-30T18:41:05.862013Z",
     "iopub.status.idle": "2022-05-30T18:41:06.635531Z",
     "shell.execute_reply": "2022-05-30T18:41:06.634851Z",
     "shell.execute_reply.started": "2022-05-30T18:41:05.862233Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f568fc2ff6e2459cbc1c2a6cdb184621",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "abortion_network =final_data.select('Legislator name').where(final_data['count_abortion'] > 0) \n",
    "abortion_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/abortion_network\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "1f72028e-9eea-4b2a-a60f-3deb4671660c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-30T18:41:06.636939Z",
     "iopub.status.busy": "2022-05-30T18:41:06.636682Z",
     "iopub.status.idle": "2022-05-30T18:41:07.472407Z",
     "shell.execute_reply": "2022-05-30T18:41:07.439731Z",
     "shell.execute_reply.started": "2022-05-30T18:41:06.636904Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "29adb6c6b73045c79ccddb7cd3dc5536",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "education_network =final_data.select('Legislator name').where(final_data['count_education'] > 0) \n",
    "education_network.write.csv(\"s3://lynette-dang-final-project-data/issue_based_network/education_network\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a8da378",
   "metadata": {},
   "source": [
    "## Reference:\n",
    "1. https://github.com/lsc4ss-s21/large-scale-personal-finance/blob/main/4_Pyspark_topic_modeling.ipynb\n",
    "2. https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/2.Text_Preprocessing_with_SparkNLP_Annotators_Transformers.ipynb#scrollTo=EgdWV7yFix8e\n",
    "3. https://medium.com/trustyou-engineering/topic-modelling-with-pyspark-and-spark-nlp-a99d063f1a6e\n",
    "4. https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.clustering.LDA.html\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark",
   "language": "",
   "name": "pysparkkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "python",
    "version": 3
   },
   "mimetype": "text/x-python",
   "name": "pyspark",
   "pygments_lexer": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}