notebooks/Document_Clustering.ipynb

{"cells":[{"cell_type":"markdown","source":["## Import useful Python packages"],"metadata":{"id":"YOrmY8FiOMwa","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"51e0f431-18c6-44e8-b07d-ade6333da085"}}},{"cell_type":"code","source":["import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nimport pyspark\nfrom pyspark.sql import *\nfrom pyspark.sql.types import *\nfrom pyspark.sql.functions import *\nfrom pyspark import SparkContext, SparkConf"],"metadata":{"id":"Fh8zPg5APmYv","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"bd8e7b26-05e8-4df8-9721-9ed64ca2c60d"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["## Check everything is ok"],"metadata":{"id":"_eRvyl2xwCV6","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"960dbf9b-1be3-46d6-aaed-5f60f59a485b"}}},{"cell_type":"code","source":["spark"],"metadata":{"id":"4fqJ5f0JE3BL","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"cd6ddbfa-c9b6-409f-9ee8-8a9ca5f1ab2e"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["sc._conf.getAll()"],"metadata":{"id":"qhTN342EEOYZ","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"649af683-866e-4a9e-b2ba-779d549d60cd"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["# **Data Acquisition**\n\n**NOTE:** This is the <code>abc-news</code> dataset available from [Kaggle](https://www.kaggle.com/therohk/million-headlines), which contains more tan **1 million** news headlines published over a period of eighteen years by the reputable Australian news source ABC ([Australian Broadcasting Corporation](http://www.abc.net.au))."],"metadata":{"id":"Mu77QbD2vC_o","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"df11164b-aafa-48ab-bd07-c1af50f17c2b"}}},{"cell_type":"markdown","source":["### Download the dataset to the local driver node's ```/tmp``` folder using ```wget```"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"749651ff-cd97-42eb-85b4-a0595e9b2f3b"}}},{"cell_type":"code","source":["%sh wget -P /tmp https://github.com/gtolomei/big-data-computing/raw/master/datasets/abc-news.csv.bz2"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"47c03730-63bd-4f6d-806f-9620413c4a0c"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["%fs ls file:/tmp/"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"36c1c162-5ecc-45fc-b968-5e7b7ea61eb1"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Move the file from local driver node's file system to DBFS"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"eed403ff-b80a-46ce-b761-d63389c97026"}}},{"cell_type":"code","source":["dbutils.fs.mv(\"file:/tmp/abc-news.csv.bz2\", \"dbfs:/bdc-2020-21/datasets/abc-news.csv.bz2\")"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"28a0b1c4-2987-403f-961f-f2fbf1c14bc9"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["%fs ls /bdc-2020-21/datasets/"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"75bba455-ddde-452e-a7b2-7cc91313d5ca"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Read dataset file into a Spark Dataframe"],"metadata":{"id":"DevlrMcPw1ZI","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"3ab8f79f-f1af-4697-bc78-e053104173d8"}}},{"cell_type":"code","source":["news_df = spark.read.load(\"dbfs:/bdc-2020-21/datasets/abc-news.csv.bz2\", \n                         format=\"csv\", \n                         sep=\",\", \n                         inferSchema=\"true\", \n                         header=\"true\"\n                         )"],"metadata":{"id":"qKi5Hd60FFcX","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f0f9699b-ea2c-4d66-8d4c-a2fd59eaf8b5"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Check the shape of the loaded dataset, i.e., number of rows and columns"],"metadata":{"id":"IPTd8ep9x74H","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"45d74a68-0ff9-4a20-8922-780b05494a6d"}}},{"cell_type":"code","source":["print(\"The shape of the dataset is {:d} rows by {:d} columns\".format(news_df.count(), len(news_df.columns)))"],"metadata":{"id":"JyRyYqeXGA4l","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6635ca26-97bc-47d1-ad9f-430b2870b0aa"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Print out the schema of the loaded dataset"],"metadata":{"id":"-WC4RPQgyEsB","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"da6b23f9-c5f9-4bb0-80e0-579b05aa1825"}}},{"cell_type":"code","source":["news_df.printSchema()"],"metadata":{"id":"Q3KhtSnvGIwG","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0e64e1a1-1a2a-4233-a7b2-2cab8834a93c"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Display the first 5 rows of the dataset"],"metadata":{"id":"0Pjib1fiylb6","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f6badcfd-0e2d-4db6-9d21-c2352f0a644d"}}},{"cell_type":"code","source":["news_df.show(5, truncate=False)"],"metadata":{"id":"a4v-Z92rGXoe","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"347630a6-de10-47c3-8cf3-095363b63680"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Count the number of duplicated news (if any)"],"metadata":{"id":"vA_0l5Rqz6lg","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"162bd3f9-eda1-453a-8eb1-6093026dc388"}}},{"cell_type":"code","source":["print(\"The total number of duplicated news are {:d} out of {:d}\".\n      format(news_df.count() - news_df.dropDuplicates(['headline_text']).count(), news_df.count()))"],"metadata":{"id":"vgKnTs5qno8M","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6bff84e7-76ff-429b-a657-31ca168a00ad"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Display the top-10 most duplicated news"],"metadata":{"id":"pIewdnUy43lH","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"cca8f7d8-c43b-4eee-b442-15e0d58243ea"}}},{"cell_type":"code","source":["news_df.groupby([\"headline_text\"]).count().sort(\"count\", ascending=False).show(10)"],"metadata":{"id":"dc85D2XDq6Fo","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fc7f1845-5039-4fd8-8c39-921bc3b85fb8"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Remove duplicate news"],"metadata":{"id":"DtcnTISuZlMh","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"1558b723-a56f-4a45-85e5-08674e893f3e"}}},{"cell_type":"code","source":["news_df = news_df.dropDuplicates([\"headline_text\"])"],"metadata":{"id":"lGHCoGwB9mBB","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"34710c82-c502-4112-8ff2-164d89f8600a"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["print(\"The total number of unique news is: {:d}\".format(news_df.count()))"],"metadata":{"id":"NlbuZU7N9vIK","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"7d0cfe78-54f2-4b72-a915-fb34746b45d4"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Check for any missing value (i.e., <code>NULL</code>) along <code>headline_text</code> column"],"metadata":{"id":"ELIYdrnk04Yu","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"68ee101c-3aa2-4851-8fc7-c90e8f3e02a3"}}},{"cell_type":"code","source":["news_df.where(col(\"headline_text\").isNull()).count()\n# Alternatively, using filter:\n# news_df.filter(news_df.headline_text.isNull()).count()"],"metadata":{"id":"5AQt_-7z1EH9","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0d24d9e8-26bf-4c5b-a8b8-2fb3fb381d11"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Show the corresponding NULL entry/ies"],"metadata":{"id":"UQQLfLzcDYWe","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"27c37ea9-45b8-40bd-ac14-7de8aa8e97f8"}}},{"cell_type":"code","source":["news_df.where(col(\"headline_text\").isNull()).show()"],"metadata":{"id":"YUTcKAUq1gGv","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"a87d4cd9-3bc6-4ab1-b45e-6c88c387eb49"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["# **Data Preprocessing**"],"metadata":{"id":"66pFGdch-BU_","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6f7fa081-0f5f-407f-be4a-89f598112616"}}},{"cell_type":"markdown","source":["In this example, we are working with text data and our ultimate goal is to cluster news into groups of coherent \"topics\" using one of the clustering algorithms we know (e.g., K-means). This is a specific task of a more general area, which is referred to as _natural language processing_ (NLP).\n\nAs **preliminary** steps of any NLP task, at least the following pipeline must be executed first:\n\n- Text cleaning:\n - Case normalization (<code>lower</code>) -> convert all text to lower case;\n - Filter out _leading_ and _trailing_ whitespaces (<code>trim</code>);\n - Filter out punctuation symbols (<code>regexp_replace</code>);\n - Filter out any internal extra whitespace resulting from the step above (<code>regexp_replace</code> + <code>trim</code>).\n- Tokenization (<code>Tokenizer</code>): splitting raw text into a list of individual _tokens_ (i.e., words), typically using whitespace as delimiter \n- Stopwords removal (<code>StopWordsRemover</code>): removing so-called _stopwords_, namely words that do not contribute to the deeper meaning of the document like \"the\", \"a\", \"me\", etc.\n- Stemming (<code>SnowballStemmer</code>): reducing each word to its root or base. For example \"fishing\", \"fished\", \"fisher\" all reduce to the stem \"fish\"."],"metadata":{"id":"pzjt71kS-ZwY","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5d93e3c5-2427-4a81-92f1-3cc77f86091a"}}},{"cell_type":"code","source":["def clean_text(df, column_name=\"headline_text\"):\n    \"\"\" \n    This function takes the raw text data and applies a standard NLP preprocessing pipeline consisting of the following steps:\n      - Text cleaning\n      - Tokenization\n      - Stopwords removal\n      - Stemming (Snowball stemmer)\n\n    parameter: dataframe\n    returns: the input dataframe along with the `cleaned_content` column as the results of the NLP preprocessing pipeline\n\n    \"\"\"\n    from pyspark.sql.functions import udf, col, lower, trim, regexp_replace\n    from pyspark.ml.feature import Tokenizer, StopWordsRemover\n    from nltk.stem.snowball import SnowballStemmer # BE SURE NLTK IS INSTALLED ON THE CLUSTER USING THE \"LIBRARIES\" TAB IN THE MENU\n\n    # Text preprocessing pipeline\n    print(\"***** Text Preprocessing Pipeline *****\\n\")\n\n    # 1. Text cleaning\n    print(\"# 1. Text Cleaning\\n\")\n    # 1.a Case normalization\n    print(\"1.a Case normalization:\")\n    lower_case_news_df = df.select(lower(col(column_name)).alias(column_name))\n    lower_case_news_df.show(10)\n    # 1.b Trimming\n    print(\"1.b Trimming:\")\n    trimmed_news_df = lower_case_news_df.select(trim(col(column_name)).alias(column_name))\n    trimmed_news_df.show(10)\n    # 1.c Filter out punctuation symbols\n    print(\"1.c Filter out punctuation:\")\n    no_punct_news_df = trimmed_news_df.select((regexp_replace(col(column_name), \"[^a-zA-Z\\\\s]\", \"\")).alias(column_name))\n    no_punct_news_df.show(10)\n    # 1.d Filter out any internal extra whitespace\n    print(\"1.d Filter out extra whitespaces:\")\n    cleaned_news_df = no_punct_news_df.select(trim(regexp_replace(col(column_name), \" +\", \" \")).alias(column_name))\n    cleaned_news_df.show(10)\n\n    # 2. Tokenization (split text into tokens)\n    print(\"# 2. Tokenization:\")\n    tokenizer = Tokenizer(inputCol=column_name, outputCol=\"tokens\")\n    tokens_df = tokenizer.transform(cleaned_news_df).cache()\n    tokens_df.show(10)\n\n    # 3. Stopwords removal\n    print(\"# 3. Stopwords removal:\")\n    stopwords_remover = StopWordsRemover(inputCol=\"tokens\", outputCol=\"terms\")\n    terms_df = stopwords_remover.transform(tokens_df).cache()\n    terms_df.show(10)\n\n    # 4. Stemming (Snowball stemmer)\n    print(\"# 4. Stemming:\")\n    stemmer = SnowballStemmer(language=\"english\")\n    stemmer_udf = udf(lambda tokens: [stemmer.stem(token) for token in tokens], ArrayType(StringType()))\n    terms_stemmed_df = terms_df.withColumn(\"terms_stemmed\", stemmer_udf(\"terms\")).cache()\n    terms_stemmed_df.show(10)\n    \n    return terms_stemmed_df"],"metadata":{"id":"prr9bVUydzmI","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"9f3c6a01-2072-4d41-ac81-ad7209afd050"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["clean_news_df = clean_text(news_df)"],"metadata":{"id":"kOITbChE0WRp","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"74666cd4-a6c0-4c5f-8b31-3dc64ddb1300"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["# **Feature Engineering**\n\nMachine learning techniques cannot work directly on text data; in fact, words must be first converted into some numerical representation that machine learning algorithms can make use of. This process is often known as _vectorization_.\n\nIn terms of vectorization, it is important to remember that it isn't merely turning a single word into a single number. While words can be transformed into numbers, an entire document can be translated into a vector. Moreover, vectors derived from text data are usually high-dimensional. This is because each dimension of the feature space will correspond to a word, and the language in the documents may have thousands of words."],"metadata":{"id":"KxYII3XAqCp7","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fda9f590-eb16-4ee7-9d3d-671f8685c2db"}}},{"cell_type":"markdown","source":["## TF-IDF\nIn information retrieval, **tf-idf** - short for term frequency-inverse document frequency - is a numerical statistic that is intended to reflect how important a word is to a document in a collection or corpus.\n\nThe tf-idf value increases proportionally to the number of times a word appears in the document and is offset by the frequency of the word in the corpus, which helps to adjust for the fact that some words appear more frequently in general."],"metadata":{"id":"e2Ck6WXpAaqs","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5a66e6a2-6edc-4db0-afed-19ed00264b96"}}},{"cell_type":"code","source":["RANDOM_SEED = 42 # used below to run the actual K-means clustering\nVOCAB_SIZE = 1000 # number of words to be retained as vocabulary\nMIN_DOC_FREQ = 10 # minimum number of documents a word has to appear in to be included in the vocabulary\nN_GRAMS = 2 # number of n-grams (if needed)\nN_FEATURES = 200 # default embedding vector size (if HashingTF or, later, Word2Vec are used)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"2a2c903c-6a4e-4602-9c6a-15f63d90a23a"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["def extract_tfidf_features(df, column_name=\"terms_stemmed\"):\n    \"\"\" \n    This fucntion takes the text data and converts it into a term frequency-inverse document frequency vector\n\n    parameter: dataframe\n    returns: dataframe with tf-idf vectors\n\n    \"\"\"\n\n    # Importing the feature transformation classes for doing TF-IDF \n    from pyspark.ml.feature import HashingTF, CountVectorizer, IDF, NGram\n    from pyspark.ml import Pipeline\n\n    ## Extracting n-grams from text\n    #ngrams = NGram(n=N_GRAMS, inputCol=column_name, outputCol=\"ngrams\")\n    #ngrams.transform(df)\n    \n    ## Creating Term Frequency Vector for each word\n    #cv = CountVectorizer(inputCol=column_name, outputCol=\"tf_features\", vocabSize=VOCAB_SIZE, minDF=MIN_DOC_FREQ)\n    #cv_model = cv.fit(df)\n    #tf_features_df = cv_model.transform(df).cache()\n\n    ## Alternatively to CountVectorizer, use HashingTF\n    #hashing_TF = HashingTF(inputCol=column_name, outputCol=\"tf_features\", numFeatures=N_FEATURES)\n    #tf_features_df = hashing_TF.transform(df).cache()\n\n    ## Carrying out Inverse Document Frequency on the TF data\n    #idf = IDF(inputCol=\"tf_features\", outputCol=\"features\")\n    #idf_model = idf.fit(tf_features_df)\n    #tf_idf_features_df = idf_model.transform(tf_features_df).cache()\n\n    # USING PIPELINE\n    #ngrams = NGram(n=N_GRAMS, inputCol=column_name, outputCol=\"ngrams\")\n    cv = CountVectorizer(inputCol=column_name, outputCol=\"tf_features\", vocabSize=VOCAB_SIZE, minDF=MIN_DOC_FREQ)\n    # hashingTF = HashingTF(inputCol=column_name, outputCol=\"tf_features\", numFeatures=N_FEATURES)\n    idf = IDF(inputCol=\"tf_features\", outputCol=\"features\")\n\n    pipeline = Pipeline(stages=[cv, idf]) # add `ngrams` and replace `cv` with `hashingTF`, if needed\n    features = pipeline.fit(df)\n    tf_idf_features_df = features.transform(df).cache()\n\n    return tf_idf_features_df"],"metadata":{"id":"13RW7AdQBeD_","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b442e961-64cf-4fb1-85e5-ff4b7f90892f"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["features = extract_tfidf_features(clean_news_df)"],"metadata":{"id":"IQprR3DjdYDY","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"ac5d18bc-bd66-4314-ac7b-c57f8e311636"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["features.select(col(\"features\")).show(10, truncate=False)"],"metadata":{"id":"OcjFWSitdday","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fc778c66-2ff1-4d32-9c89-e35bc56eb93b"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["clean_news_df.show(5, truncate=False)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f7687e78-0796-474e-a626-3b5be25c3b65"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Check and remove any possible zero-length vector"],"metadata":{"id":"r_AVPZudEfZv","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"ed47012c-c7c7-467a-a8c1-631ca80af8a3"}}},{"cell_type":"code","source":["@udf(\"long\")\ndef num_nonzeros(v):\n    return v.numNonzeros()"],"metadata":{"id":"gus3IFnA7JAC","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"9d673136-8acc-4c51-bca6-4c14a96f2bf4"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["#### Check if there is any zero-length vector"],"metadata":{"id":"qFYhRWY_PwHC","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"1c171e99-1a4d-4ba8-bab1-6690c9e42b8a"}}},{"cell_type":"code","source":["print(\"Total n. of zero-length vectors: {:d}\".\n      format(features.where(num_nonzeros(\"features\") == 0).count()))"],"metadata":{"id":"VyZj5ONJPuth","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"547397c8-3acf-4b03-b9a6-eddc86826372"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["#### Remove zero-length vector(s)"],"metadata":{"id":"23FneXsvQJaZ","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"3c0a0f55-49b2-4255-9b5d-541cd764e9d4"}}},{"cell_type":"code","source":["features = features.where(num_nonzeros(\"features\") > 0)"],"metadata":{"id":"8-YCkcO4PrWP","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"482208cd-9188-4a87-9210-af5622e90e9a"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["#### Double-check there is no more zero-length vector"],"metadata":{"id":"MEAtctNbQNeG","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f1f79f17-ca74-43b8-9d43-7e035c00360d"}}},{"cell_type":"code","source":["print(\"Total n. of zero-length vectors (after removal): {:d}\".\n      format(features.where(num_nonzeros(\"features\") == 0).count()))"],"metadata":{"id":"7TNJ35M0GoBD","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"c8e59a30-9c82-4c60-9689-27997d398d9f"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["# **K-means Clustering**"],"metadata":{"id":"ZE2DTA2NdfSA","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"8ecaf894-4d2d-4c04-a2ce-edb100633384"}}},{"cell_type":"code","source":["N_CLUSTERS = 10 # number of output clusters (K)\nDISTANCE_MEASURE = \"euclidean\" # alternatively, \"cosine\"\nMAX_ITERATIONS = 100 # maximum number of iterations of K-means EM algorithm\nTOLERANCE = 0.000001 # tolerance between consecutive centroid updates (i.e., another stopping criterion)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5f7d24c2-02d1-4bf6-8fc6-961f24b1edfa"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Function used for running K-means"],"metadata":{"id":"jPdq2yVE9LzC","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"0282028d-ebb8-448a-9af3-b2fd73817a29"}}},{"cell_type":"code","source":["def k_means(dataset, \n            n_clusters, \n            distance_measure=DISTANCE_MEASURE, \n            max_iter=MAX_ITERATIONS, \n            tol=TOLERANCE,\n            features_col=\"features\", \n            prediction_col=\"cluster\", \n            random_seed=RANDOM_SEED):\n  \n  from pyspark.ml.clustering import KMeans\n\n  print(\"\"\"Training K-means clustering using the following parameters: \n  - K (n. of clusters) = {:d}\n  - max_iter (max n. of iterations) = {:d}\n  - distance measure = {:s}\n  - random seed = {:d}\n  \"\"\".format(n_clusters, max_iter, distance_measure, random_seed))\n  # Train a K-means model\n  kmeans = KMeans(featuresCol=features_col, \n                   predictionCol=prediction_col, \n                   k=n_clusters, \n                   initMode=\"k-means||\", \n                   initSteps=5, \n                   tol=tol, \n                   maxIter=max_iter, \n                   seed=random_seed, \n                   distanceMeasure=distance_measure)\n  model = kmeans.fit(dataset)\n\n  # Make clusters\n  clusters_df = model.transform(dataset).cache()\n\n  return model, clusters_df"],"metadata":{"id":"JP1deqVUurtB","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f66713e9-eda3-493a-95dd-8be58e87a103"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Run K-means by calling the function above"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"85ea7fbd-2fd2-48d5-8bdb-2506199ffe4b"}}},{"cell_type":"code","source":["model, clusters_df = k_means(features, N_CLUSTERS, max_iter=MAX_ITERATIONS, distance_measure=DISTANCE_MEASURE)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"39f7abd8-f38d-4037-aa5c-e6f5f2e060b5"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Function used to evaluate obtained clusters (Silhouette Coefficient)"],"metadata":{"id":"pi3FUnyx9ZWi","colab_type":"text","application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"1ee2e0c6-3d5c-4f1e-8f87-f4b93b303fcd"}}},{"cell_type":"code","source":["def evaluate_k_means(clusters, \n                     metric_name=\"silhouette\", \n                     distance_measure=\"squaredEuclidean\", # cosine\n                     prediction_col=\"cluster\"\n                     ):\n  \n  from pyspark.ml.evaluation import ClusteringEvaluator\n  \n  # Evaluate clustering by computing Silhouette score\n  evaluator = ClusteringEvaluator(metricName=metric_name,\n                                  distanceMeasure=distance_measure, \n                                  predictionCol=prediction_col\n                                  )\n\n  return evaluator.evaluate(clusters)"],"metadata":{"id":"nZTOi7TWvcXJ","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"03ae8198-3395-483f-9980-2b9313cd4f40"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Evaluate clustering (Silhouette Coefficient)\n\nWe can evaluate the clustering we just run using K=10 clusters"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b66dd34c-25d7-49d2-9061-f7a04577e0e1"}}},{"cell_type":"code","source":["evaluate_k_means(clusters_df, distance_measure=\"squaredEuclidean\")"],"metadata":{"id":"TYu_vheKwTc0","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f5de60aa-a6dc-40c3-93a9-944d880ae37d"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["clusters_df.show(5)"],"metadata":{"id":"Q_1-52c-HtVh","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"e1392f90-6fe4-455e-80ba-2e6b4f554564"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["clusters_df.groupBy(\"cluster\").count().sort(\"cluster\").show()"],"metadata":{"id":"O2MLijHVJHIi","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"9ce32f96-5257-4cc1-9f50-e9bb48978e32"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["# Get unique values in the grouping column\nclusters = sorted([x[0] for x in clusters_df.select(\"cluster\").distinct().collect()])\nprint(\"Cluster IDs: [{:s}]\".format(\", \".join([str(c) for c in clusters])))\n\n# Create a filtered DataFrame for each group in a list comprehension\ncluster_list = [clusters_df.where(clusters_df.cluster == x) for x in clusters]\n\n# Show the results (first 5 cluters)\nfor x_id, x in enumerate(cluster_list):\n  print(\"Showing the first 10 records of cluster ID #{:d}\".format(x_id))\n  x.select([\"cluster\", \"headline_text\"]).show(10, truncate=True)"],"metadata":{"id":"Qqft7DRcHUoG","colab_type":"code","colab":{},"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"381dc391-8154-4e63-8d08-c42c69851fe5"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["# **Feature Learning**\n\nInstead of using TF-IDF, we can _learn_ how to extract features using [Word2Vec](https://code.google.com/p/word2vec/).\n\nWord2Vec computes distributed vector representation of words. The main advantage of the distributed representations is that similar words are close in the vector space, which makes generalization to novel patterns easier and model estimation more robust. Distributed vector representation is showed to be useful in many natural language processing applications such as named entity recognition, disambiguation, parsing, tagging and machine translation."],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"fc73a377-0982-48c6-b4e5-f4ef8fa5b8a7"}}},{"cell_type":"code","source":["EMBEDDING_SIZE = 150 # size of embedding Word2Vec vectors"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"472a3313-cebb-481a-876e-4825b8925bf0"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["def extract_w2v_features(df, column_name=\"terms\"):\n  from pyspark.ml.feature import Word2Vec\n  \n  word2vec = Word2Vec(vectorSize=EMBEDDING_SIZE, minCount=5, inputCol=column_name, outputCol=\"features\", seed=RANDOM_SEED)\n  model = word2vec.fit(df)\n  features = model.transform(df).cache()\n  \n  return model, features"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"19f83ea4-180c-470b-b75c-25903a531004"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["model, w2v_features = extract_w2v_features(clean_news_df)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"3ce118c6-4c8d-496b-8725-9727a9eda761"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["w2v_features.show(truncate=False)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"4b198dba-7feb-4a51-8f51-a4cc7fae1b53"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["vecs = model.getVectors()\nsyms = model.findSynonyms(\"doctor\", 2)\nsyms.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"a5c5681b-b66f-4ee0-883c-5c83b82a220d"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Find the \"best\" value of K\n\nWe can use the \"elbow method\" to find the best value of K."],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"40fd1aeb-810f-434e-813c-bf5d946c8b3c"}}},{"cell_type":"code","source":["K_MIN = 2\nK_MAX = 18\nSTEP = 2"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"10b52bb7-ba04-4393-aa13-64c1ee7f3ab6"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["def elbow_method(data, k_min=K_MIN, k_max=K_MAX, step=STEP, max_iter=MAX_ITERATIONS, distance_measure=DISTANCE_MEASURE):\n  results = []\n  for k in range(k_min, k_max, step):\n    model, clusters_df = k_means(data, k, max_iter=max_iter, distance_measure=distance_measure)\n    results.append([k, model.summary.trainingCost])\n\n  return pd.DataFrame(results, columns = ['K', 'SSE'])"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"2189a0c4-e9cd-4cea-9b4b-aba8a906ae15"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["# Get results from elbow method\nelbow_results = elbow_method(w2v_features)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"4852e536-4e42-40dd-8886-c8104013d462"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"markdown","source":["### Plot the results of \"elbow method\""],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b7485ad6-2fdc-40e2-a9ce-9fb0fc041154"}}},{"cell_type":"code","source":["from matplotlib.ticker import MaxNLocator\n\ndef plot_elbow(results_df):\n  fig, ax = plt.subplots(1,1, figsize =(8,6))\n  _ = sns.lineplot(data=results_df, x=\"K\", y=\"SSE\", ax=ax)\n  ax.set_xlabel('K (#clusters)')\n  ax.set_ylabel('SSE')\n  ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n  plt.show()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d8e0c5ca-ff58-43ae-9f56-b5454167568e"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0},{"cell_type":"code","source":["plot_elbow(elbow_results)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"4cf418b9-c1b7-4bdb-8a29-8e800c65118c"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"type":"ipynbError","data":"","errorSummary":"","arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>"]}}],"execution_count":0}],"metadata":{"colab":{"name":"Lecture_06_Document_Clustering.ipynb","collapsed_sections":[],"machine_shape":"hm","authorship_tag":"ABX9TyOu2J0kmKvyuw0fUk5AYrpg","provenance":[{"file_id":"1-fYE7uaSyOk1LkQ5mF-_2PT2uIYRIVXn","timestamp":1581962053137}]},"kernelspec":{"name":"python3","display_name":"Python 3"},"application/vnd.databricks.v1+notebook":{"notebookName":"Document_Clustering","dashboards":[],"notebookMetadata":{"pythonIndentUnit":2},"language":"python","widgets":{},"notebookOrigID":2063663459813559}},"nbformat":4,"nbformat_minor":0}