feature-engineering-and-syntactic-similarity.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-28T15:33:40.171177Z","iopub.execute_input":"2023-12-28T15:33:40.172036Z","iopub.status.idle":"2023-12-28T15:33:41.302470Z","shell.execute_reply.started":"2023-12-28T15:33:40.171976Z","shell.execute_reply":"2023-12-28T15:33:41.300991Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"/kaggle/input/million-headlines/abcnews-date-text.csv\n","output_type":"stream"}]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/million-headlines/abcnews-date-text.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:33:43.342474Z","iopub.execute_input":"2023-12-28T15:33:43.343167Z","iopub.status.idle":"2023-12-28T15:33:45.921529Z","shell.execute_reply.started":"2023-12-28T15:33:43.343124Z","shell.execute_reply":"2023-12-28T15:33:45.920249Z"},"trusted":true},"execution_count":2,"outputs":[{"execution_count":2,"output_type":"execute_result","data":{"text/plain":"   publish_date                                      headline_text\n0      20030219  aba decides against community broadcasting lic...\n1      20030219     act fire witnesses must be aware of defamation\n2      20030219     a g calls for infrastructure protection summit\n3      20030219           air nz staff in aust strike for pay rise\n4      20030219      air nz strike to affect australian travellers","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>publish_date</th>\n      <th>headline_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>20030219</td>\n      <td>aba decides against community broadcasting lic...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>20030219</td>\n      <td>act fire witnesses must be aware of defamation</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>20030219</td>\n      <td>a g calls for infrastructure protection summit</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>20030219</td>\n      <td>air nz staff in aust strike for pay rise</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>20030219</td>\n      <td>air nz strike to affect australian travellers</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:33:45.923856Z","iopub.execute_input":"2023-12-28T15:33:45.925376Z","iopub.status.idle":"2023-12-28T15:33:46.103565Z","shell.execute_reply.started":"2023-12-28T15:33:45.925312Z","shell.execute_reply":"2023-12-28T15:33:46.101873Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 1244184 entries, 0 to 1244183\nData columns (total 2 columns):\n #   Column         Non-Null Count    Dtype \n---  ------         --------------    ----- \n 0   publish_date   1244184 non-null  int64 \n 1   headline_text  1244184 non-null  object\ndtypes: int64(1), object(1)\nmemory usage: 19.0+ MB\n","output_type":"stream"}]},{"cell_type":"code","source":"df.loc[0, 'headline_text']","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:33:46.105161Z","iopub.execute_input":"2023-12-28T15:33:46.105628Z","iopub.status.idle":"2023-12-28T15:33:46.116584Z","shell.execute_reply.started":"2023-12-28T15:33:46.105592Z","shell.execute_reply":"2023-12-28T15:33:46.115023Z"},"trusted":true},"execution_count":4,"outputs":[{"execution_count":4,"output_type":"execute_result","data":{"text/plain":"'aba decides against community broadcasting licence'"},"metadata":{}}]},{"cell_type":"markdown","source":"Get a sample data to quickly iterate on approaches","metadata":{}},{"cell_type":"code","source":"sample_df = df.sample(frac=0.01)\nsample_df.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:33:46.839515Z","iopub.execute_input":"2023-12-28T15:33:46.839908Z","iopub.status.idle":"2023-12-28T15:33:46.912272Z","shell.execute_reply.started":"2023-12-28T15:33:46.839877Z","shell.execute_reply":"2023-12-28T15:33:46.910856Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"<class 'pandas.core.frame.DataFrame'>\nIndex: 12442 entries, 555280 to 423367\nData columns (total 2 columns):\n #   Column         Non-Null Count  Dtype \n---  ------         --------------  ----- \n 0   publish_date   12442 non-null  int64 \n 1   headline_text  12442 non-null  object\ndtypes: int64(1), object(1)\nmemory usage: 291.6+ KB\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Bag of Words Model\n\nFirst thing we will do is to apply a basic count vectorizer to generate one hot encoded represantation of our vocabulary and data","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\n\ndef count_vectorizer_fn(**kwargs):\n    cv = CountVectorizer(**kwargs)\n    dt = cv.fit_transform(sample_df['headline_text'])\n    return dt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:33:48.760667Z","iopub.execute_input":"2023-12-28T15:33:48.761175Z","iopub.status.idle":"2023-12-28T15:33:50.189688Z","shell.execute_reply.started":"2023-12-28T15:33:48.761135Z","shell.execute_reply":"2023-12-28T15:33:50.187974Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"count_vectorizer_fn()","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:33:51.375114Z","iopub.execute_input":"2023-12-28T15:33:51.375619Z","iopub.status.idle":"2023-12-28T15:33:51.756321Z","shell.execute_reply.started":"2023-12-28T15:33:51.375582Z","shell.execute_reply":"2023-12-28T15:33:51.754455Z"},"trusted":true},"execution_count":7,"outputs":[{"execution_count":7,"output_type":"execute_result","data":{"text/plain":"<12442x14416 sparse matrix of type '<class 'numpy.int64'>'\n\twith 81039 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"12k rows and 14k vocab size for the sample. This is not exactly scalable to bigger data as vocab size would get bigger. We can try to optimize our text as we didn't do much of that now","metadata":{}},{"cell_type":"markdown","source":"## Reducing Feature Dimension","metadata":{}},{"cell_type":"markdown","source":"### Stopwords","metadata":{}},{"cell_type":"code","source":"from spacy.lang.en.stop_words import STOP_WORDS as stopwords\n\n\nprint(len(stopwords))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:03.352459Z","iopub.execute_input":"2023-12-28T15:34:03.353431Z","iopub.status.idle":"2023-12-28T15:34:10.044298Z","shell.execute_reply.started":"2023-12-28T15:34:03.353386Z","shell.execute_reply":"2023-12-28T15:34:10.042611Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"326\n","output_type":"stream"}]},{"cell_type":"code","source":"count_vectorizer_fn(stop_words=list(stopwords))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:10.047080Z","iopub.execute_input":"2023-12-28T15:34:10.047716Z","iopub.status.idle":"2023-12-28T15:34:10.328192Z","shell.execute_reply.started":"2023-12-28T15:34:10.047680Z","shell.execute_reply":"2023-12-28T15:34:10.325825Z"},"trusted":true},"execution_count":9,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:409: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['ll', 've'] not in stop_words.\n  warnings.warn(\n","output_type":"stream"},{"execution_count":9,"output_type":"execute_result","data":{"text/plain":"<12442x14198 sparse matrix of type '<class 'numpy.int64'>'\n\twith 65248 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"This didn't reduce out dataset size much as we still have about 14k vocab size","metadata":{}},{"cell_type":"markdown","source":"### Minimum Document Frequency","metadata":{}},{"cell_type":"code","source":"count_vectorizer_fn(stop_words=list(stopwords), min_df=2)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:10.330217Z","iopub.execute_input":"2023-12-28T15:34:10.331363Z","iopub.status.idle":"2023-12-28T15:34:10.546024Z","shell.execute_reply.started":"2023-12-28T15:34:10.331313Z","shell.execute_reply":"2023-12-28T15:34:10.544737Z"},"trusted":true},"execution_count":10,"outputs":[{"execution_count":10,"output_type":"execute_result","data":{"text/plain":"<12442x6889 sparse matrix of type '<class 'numpy.int64'>'\n\twith 57939 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"code","source":"count_vectorizer_fn(stop_words=list(stopwords), min_df=0.0001)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:10.548767Z","iopub.execute_input":"2023-12-28T15:34:10.549153Z","iopub.status.idle":"2023-12-28T15:34:10.766253Z","shell.execute_reply.started":"2023-12-28T15:34:10.549124Z","shell.execute_reply":"2023-12-28T15:34:10.764692Z"},"trusted":true},"execution_count":11,"outputs":[{"execution_count":11,"output_type":"execute_result","data":{"text/plain":"<12442x6889 sparse matrix of type '<class 'numpy.int64'>'\n\twith 57939 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"Setting a min_df reduces our vocab size by half!! which is a big improvement in terms of scalability ","metadata":{}},{"cell_type":"markdown","source":"### Maximum Document Frequency\n\nWe might have a corpus with a lot of repeating terms so we can remove them via max_df parameter.","metadata":{}},{"cell_type":"code","source":"count_vectorizer_fn(stop_words=list(stopwords), max_df=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:10.768895Z","iopub.execute_input":"2023-12-28T15:34:10.769481Z","iopub.status.idle":"2023-12-28T15:34:10.995883Z","shell.execute_reply.started":"2023-12-28T15:34:10.769435Z","shell.execute_reply":"2023-12-28T15:34:10.994833Z"},"trusted":true},"execution_count":12,"outputs":[{"execution_count":12,"output_type":"execute_result","data":{"text/plain":"<12442x14198 sparse matrix of type '<class 'numpy.int64'>'\n\twith 65248 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"Okay that didnt reduce anything? This is because of stop words usage. Using stop words is often the better choice.","metadata":{}},{"cell_type":"code","source":"count_vectorizer_fn(max_df=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:10.997323Z","iopub.execute_input":"2023-12-28T15:34:10.998295Z","iopub.status.idle":"2023-12-28T15:34:11.219476Z","shell.execute_reply.started":"2023-12-28T15:34:10.998256Z","shell.execute_reply":"2023-12-28T15:34:11.218284Z"},"trusted":true},"execution_count":13,"outputs":[{"execution_count":13,"output_type":"execute_result","data":{"text/plain":"<12442x14413 sparse matrix of type '<class 'numpy.int64'>'\n\twith 75808 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"### N-Grams\n\nWe could apply a simple logic of using group of words togeter in the vocabulary. This would of course increase the size of data but would provide good information to us","metadata":{}},{"cell_type":"code","source":"count_vectorizer_fn(stop_words=list(stopwords), min_df=2, ngram_range=(1, 2))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:29.615118Z","iopub.execute_input":"2023-12-28T15:34:29.615592Z","iopub.status.idle":"2023-12-28T15:34:30.056302Z","shell.execute_reply.started":"2023-12-28T15:34:29.615557Z","shell.execute_reply":"2023-12-28T15:34:30.054835Z"},"trusted":true},"execution_count":14,"outputs":[{"execution_count":14,"output_type":"execute_result","data":{"text/plain":"<12442x9763 sparse matrix of type '<class 'numpy.int64'>'\n\twith 66475 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"code","source":"count_vectorizer_fn(stop_words=list(stopwords), min_df=2, ngram_range=(1, 3))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:30.059455Z","iopub.execute_input":"2023-12-28T15:34:30.060084Z","iopub.status.idle":"2023-12-28T15:34:30.678603Z","shell.execute_reply.started":"2023-12-28T15:34:30.060036Z","shell.execute_reply":"2023-12-28T15:34:30.676827Z"},"trusted":true},"execution_count":15,"outputs":[{"execution_count":15,"output_type":"execute_result","data":{"text/plain":"<12442x10073 sparse matrix of type '<class 'numpy.int64'>'\n\twith 67236 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"## TF-IDF Models","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:35.828784Z","iopub.execute_input":"2023-12-28T15:34:35.829313Z","iopub.status.idle":"2023-12-28T15:34:35.834872Z","shell.execute_reply.started":"2023-12-28T15:34:35.829277Z","shell.execute_reply":"2023-12-28T15:34:35.833847Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(stopwords), min_df=2, ngram_range=(1, 2))\ndt = tfidf.fit_transform(sample_df['headline_text'])\ndt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:34:37.796071Z","iopub.execute_input":"2023-12-28T15:34:37.796605Z","iopub.status.idle":"2023-12-28T15:34:38.236450Z","shell.execute_reply.started":"2023-12-28T15:34:37.796568Z","shell.execute_reply":"2023-12-28T15:34:38.235054Z"},"trusted":true},"execution_count":17,"outputs":[{"execution_count":17,"output_type":"execute_result","data":{"text/plain":"<12442x9763 sparse matrix of type '<class 'numpy.float64'>'\n\twith 66475 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"## Lemmas\n\nWe can apply some linguistic techniques to improve our features. We will be adding lemmatized words. News headlines applying the lemmatization probably won't lose information value\n","metadata":{}},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\nimport spacy\n\n\nnlp = spacy.load(\"en_core_web_sm\")\nnouns_adjectives_verbs = [\"NOUN\", \"PROPN\", \"ADJ\", \"ADV\", \"VERB\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:50:20.674070Z","iopub.execute_input":"2023-12-28T15:50:20.674728Z","iopub.status.idle":"2023-12-28T15:50:22.133179Z","shell.execute_reply.started":"2023-12-28T15:50:20.674681Z","shell.execute_reply":"2023-12-28T15:50:22.131672Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"for i, row in tqdm(sample_df.iterrows(), total=len(sample_df)):\n    doc = nlp(str(row[\"headline_text\"]))\n    sample_df.at[i, \"lemmas\"] = \" \".join([token.lemma_ for token in doc])\n    sample_df.at[i, \"nav\"] = \" \".join([token.lemma_ for token in doc if token.pos_ in nouns_adjectives_verbs])","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:51:52.914899Z","iopub.execute_input":"2023-12-28T15:51:52.915456Z","iopub.status.idle":"2023-12-28T15:53:58.439180Z","shell.execute_reply.started":"2023-12-28T15:51:52.915419Z","shell.execute_reply":"2023-12-28T15:53:58.437355Z"},"trusted":true},"execution_count":22,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/12442 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6b6a18b87dea49c38a01c2991d629f93"}},"metadata":{}}]},{"cell_type":"markdown","source":"### Using Lemmas instead of Words for Vectorizing","metadata":{}},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(stopwords))\ndt = tfidf.fit_transform(sample_df[\"lemmas\"].map(str))\ndt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:55:45.507700Z","iopub.execute_input":"2023-12-28T15:55:45.508351Z","iopub.status.idle":"2023-12-28T15:55:45.750580Z","shell.execute_reply.started":"2023-12-28T15:55:45.508304Z","shell.execute_reply":"2023-12-28T15:55:45.749079Z"},"trusted":true},"execution_count":23,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:409: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['ll', 've'] not in stop_words.\n  warnings.warn(\n","output_type":"stream"},{"execution_count":23,"output_type":"execute_result","data":{"text/plain":"<12442x11544 sparse matrix of type '<class 'numpy.float64'>'\n\twith 64085 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Limit Word Types\n\nFocusing on set of words such as nouns could be better","metadata":{}},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(stopwords))\ndt = tfidf.fit_transform(sample_df[\"nav\"].map(str))\ndt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T15:58:08.639384Z","iopub.execute_input":"2023-12-28T15:58:08.640864Z","iopub.status.idle":"2023-12-28T15:58:08.874464Z","shell.execute_reply.started":"2023-12-28T15:58:08.640768Z","shell.execute_reply":"2023-12-28T15:58:08.872651Z"},"trusted":true},"execution_count":24,"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"<12442x11265 sparse matrix of type '<class 'numpy.float64'>'\n\twith 62853 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Remove Most Common Words\n\nLet's use the most common 10,000 words to remove it from our vocabulary","metadata":{}},{"cell_type":"code","source":"top_10000 = pd.read_csv(\"https://raw.githubusercontent.com/first20hours/google-10000-english/master/google-10000-english.txt\", header=None)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:02:34.749540Z","iopub.execute_input":"2023-12-28T16:02:34.751528Z","iopub.status.idle":"2023-12-28T16:02:35.022836Z","shell.execute_reply.started":"2023-12-28T16:02:34.751467Z","shell.execute_reply":"2023-12-28T16:02:35.021251Z"},"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(set(top_10000.iloc[:, 0].values)))\ndt = tfidf.fit_transform(sample_df[\"nav\"].map(str))\ndt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:02:37.451345Z","iopub.execute_input":"2023-12-28T16:02:37.451901Z","iopub.status.idle":"2023-12-28T16:02:37.654024Z","shell.execute_reply.started":"2023-12-28T16:02:37.451854Z","shell.execute_reply":"2023-12-28T16:02:37.652455Z"},"trusted":true},"execution_count":26,"outputs":[{"execution_count":26,"output_type":"execute_result","data":{"text/plain":"<12442x7004 sparse matrix of type '<class 'numpy.float64'>'\n\twith 15768 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Adding Context via N-Gram","metadata":{}},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(set(top_10000.iloc[:, 0].values)), min_df=2, ngram_range=(1, 2))\ndt = tfidf.fit_transform(sample_df[\"nav\"].map(str))\ndt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:03:35.531836Z","iopub.execute_input":"2023-12-28T16:03:35.532384Z","iopub.status.idle":"2023-12-28T16:03:35.766252Z","shell.execute_reply.started":"2023-12-28T16:03:35.532346Z","shell.execute_reply":"2023-12-28T16:03:35.764915Z"},"trusted":true},"execution_count":27,"outputs":[{"execution_count":27,"output_type":"execute_result","data":{"text/plain":"<12442x2536 sparse matrix of type '<class 'numpy.float64'>'\n\twith 11470 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"markdown","source":"## Syntactic Similarity in the Dataset\n\nLet's take a look at finding similar documents in our dataset","metadata":{}},{"cell_type":"markdown","source":"### Finding Most Similar Headlines to a Made-Up Headline\n\nImage you want to find a headline in our data  that matches to headline we remember. Let's try to do this","metadata":{}},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(stopwords), min_df=2)\ndt = tfidf.fit_transform(sample_df[\"lemmas\"].map(str))\ndt","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:10:09.249946Z","iopub.execute_input":"2023-12-28T16:10:09.250571Z","iopub.status.idle":"2023-12-28T16:10:09.469638Z","shell.execute_reply.started":"2023-12-28T16:10:09.250532Z","shell.execute_reply":"2023-12-28T16:10:09.468358Z"},"trusted":true},"execution_count":28,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:409: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['ll', 've'] not in stop_words.\n  warnings.warn(\n","output_type":"stream"},{"execution_count":28,"output_type":"execute_result","data":{"text/plain":"<12442x5709 sparse matrix of type '<class 'numpy.float64'>'\n\twith 58250 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\n\nmade_up = tfidf.transform([\"australia and new zealand discuss optimal apple size\"])\n\nsim = cosine_similarity(made_up, dt)\nsim[0]","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:12:44.061572Z","iopub.execute_input":"2023-12-28T16:12:44.062168Z","iopub.status.idle":"2023-12-28T16:12:44.178318Z","shell.execute_reply.started":"2023-12-28T16:12:44.062128Z","shell.execute_reply":"2023-12-28T16:12:44.176884Z"},"trusted":true},"execution_count":29,"outputs":[{"execution_count":29,"output_type":"execute_result","data":{"text/plain":"array([0.        , 0.07257882, 0.        , ..., 0.        , 0.        ,\n       0.        ])"},"metadata":{}}]},{"cell_type":"code","source":"sample_df.iloc[np.argsort(sim[0])[::-1][0:5]][[\"publish_date\", \"lemmas\"]]","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:13:13.332831Z","iopub.execute_input":"2023-12-28T16:13:13.333640Z","iopub.status.idle":"2023-12-28T16:13:13.354109Z","shell.execute_reply.started":"2023-12-28T16:13:13.333563Z","shell.execute_reply":"2023-12-28T16:13:13.352497Z"},"trusted":true},"execution_count":30,"outputs":[{"execution_count":30,"output_type":"execute_result","data":{"text/plain":"         publish_date                                             lemmas\n633450       20110818              federal push to ban new zealand apple\n552450       20100724           new headache for apple as iphone 4 delay\n1238723      20210722    fire the size of los angeles rip through oregon\n626690       20110714                      new label to hit aussie apple\n1202108      20200610  new zealand art music open up impact on austra...","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>publish_date</th>\n      <th>lemmas</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>633450</th>\n      <td>20110818</td>\n      <td>federal push to ban new zealand apple</td>\n    </tr>\n    <tr>\n      <th>552450</th>\n      <td>20100724</td>\n      <td>new headache for apple as iphone 4 delay</td>\n    </tr>\n    <tr>\n      <th>1238723</th>\n      <td>20210722</td>\n      <td>fire the size of los angeles rip through oregon</td>\n    </tr>\n    <tr>\n      <th>626690</th>\n      <td>20110714</td>\n      <td>new label to hit aussie apple</td>\n    </tr>\n    <tr>\n      <th>1202108</th>\n      <td>20200610</td>\n      <td>new zealand art music open up impact on austra...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Finding the Two Most Similar Documents in a Large Corpus\n\nOur main dataset has about 1m rows and converting to vectorized data and doing the dot product to compute cosine similarity might be a start however, dot product of 1m with 1m would be more than 1 trillion rows of data which is not RAM efficient. We know that similarity matrix has duplicated entries (a and b vs b and a) which means our final similartiy matrix, we can skip computation of half of rows which reduces the ram size by 50% but now we have a compute issue at hand cuz we want to skip certain computations. To do that in a vectorized way, we can compute the similarity matrix in a submatrix fashion. Meaning we would take a subset, do similarity computation instead of going through the data at once. ","metadata":{}},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words=list(stopwords), ngram_range=(1,2), min_df=2, norm='l2')\ndt = tfidf.fit_transform(sample_df[\"headline_text\"])","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:22:21.387395Z","iopub.execute_input":"2023-12-28T16:22:21.387891Z","iopub.status.idle":"2023-12-28T16:22:21.798431Z","shell.execute_reply.started":"2023-12-28T16:22:21.387851Z","shell.execute_reply":"2023-12-28T16:22:21.797052Z"},"trusted":true},"execution_count":31,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/sklearn/feature_extraction/text.py:409: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['ll', 've'] not in stop_words.\n  warnings.warn(\n","output_type":"stream"}]},{"cell_type":"markdown","source":"#### Timing Cosing Similarity","metadata":{}},{"cell_type":"code","source":"%%time\ncosine_similarity(dt[0:10000], dt[0:10000], dense_output=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:22:26.626301Z","iopub.execute_input":"2023-12-28T16:22:26.627324Z","iopub.status.idle":"2023-12-28T16:22:26.680246Z","shell.execute_reply.started":"2023-12-28T16:22:26.627282Z","shell.execute_reply":"2023-12-28T16:22:26.678703Z"},"trusted":true},"execution_count":32,"outputs":[{"name":"stdout","text":"CPU times: user 30.9 ms, sys: 12 ms, total: 43 ms\nWall time: 41.2 ms\n","output_type":"stream"},{"execution_count":32,"output_type":"execute_result","data":{"text/plain":"<10000x10000 sparse matrix of type '<class 'numpy.float64'>'\n\twith 1320492 stored elements in Compressed Sparse Row format>"},"metadata":{}}]},{"cell_type":"code","source":"%%time\nr = cosine_similarity(dt[0:10000], dt[0:10000])\nr[r > 0.9999] = 0\nprint(np.argmax(r))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:22:40.429209Z","iopub.execute_input":"2023-12-28T16:22:40.430026Z","iopub.status.idle":"2023-12-28T16:22:42.763383Z","shell.execute_reply.started":"2023-12-28T16:22:40.429988Z","shell.execute_reply":"2023-12-28T16:22:42.761929Z"},"trusted":true},"execution_count":33,"outputs":[{"name":"stdout","text":"43206829\nCPU times: user 569 ms, sys: 1.76 s, total: 2.33 s\nWall time: 2.33 s\n","output_type":"stream"}]},{"cell_type":"code","source":"%%time\nr = cosine_similarity(dt[0:10000], dt[0:10000], dense_output=False)\nr[r > 0.9999] = 0\nprint(np.argmax(r))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:23:11.756831Z","iopub.execute_input":"2023-12-28T16:23:11.757378Z","iopub.status.idle":"2023-12-28T16:23:12.235144Z","shell.execute_reply.started":"2023-12-28T16:23:11.757336Z","shell.execute_reply":"2023-12-28T16:23:12.233129Z"},"trusted":true},"execution_count":34,"outputs":[{"name":"stdout","text":"43206829\nCPU times: user 370 ms, sys: 101 ms, total: 471 ms\nWall time: 470 ms\n","output_type":"stream"}]},{"cell_type":"markdown","source":"#### Timing Dot Product","metadata":{}},{"cell_type":"code","source":"%%time\nr = np.dot(dt[0:10000], np.transpose(dt[0:10000]))\nr[r > 0.9999] = 0\nprint(np.argmax(r))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T16:24:14.627434Z","iopub.execute_input":"2023-12-28T16:24:14.627985Z","iopub.status.idle":"2023-12-28T16:24:14.996012Z","shell.execute_reply.started":"2023-12-28T16:24:14.627950Z","shell.execute_reply":"2023-12-28T16:24:14.995069Z"},"trusted":true},"execution_count":35,"outputs":[{"name":"stdout","text":"43206829\nCPU times: user 335 ms, sys: 29 ms, total: 364 ms\nWall time: 362 ms\n","output_type":"stream"}]},{"cell_type":"markdown","source":"#### Batch Data","metadata":{}},{"cell_type":"code","source":"# there are \"test\" headlines in the corpus\nstopwords.add(\"test\")\ntfidf = TfidfVectorizer(stop_words=list(stopwords), ngram_range=(1,2), min_df=2, norm='l2')\ndt = tfidf.fit_transform(df[\"headline_text\"])     # Use the whole data","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:11:39.391354Z","iopub.execute_input":"2023-12-28T17:11:39.391906Z","iopub.status.idle":"2023-12-28T17:12:23.566288Z","shell.execute_reply.started":"2023-12-28T17:11:39.391867Z","shell.execute_reply":"2023-12-28T17:12:23.564680Z"},"trusted":true},"execution_count":49,"outputs":[]},{"cell_type":"code","source":"%%time\nbatch = 10000\nmax_sim = 0.0\nmax_a = None\nmax_b = None\n\nfor a in tqdm(range(0, dt.shape[0], batch)):\n    for b in range(0, a+batch, batch):\n        # r = np.dot(dt[a:a+batch], np.transpose(dt[b:b+batch]))\n        r = cosine_similarity(dt[a:a+batch], dt[b:b+batch], dense_output=False)\n        # eliminate identical vectors\n        # by setting their similarity to np.nan which gets sorted out\n        r[r > 0.9999] = 0\n        sim = r.max()\n        if sim > max_sim:\n            # argmax returns a single value which we have to \n            # map to the two dimensions            \n            (max_a, max_b) = np.unravel_index(np.argmax(r), r.shape)\n            # adjust offsets in corpus (this is a submatrix)\n            max_a += a\n            max_b += b\n            max_sim = sim","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:12:23.568681Z","iopub.execute_input":"2023-12-28T17:12:23.569162Z","iopub.status.idle":"2023-12-28T17:23:25.532997Z","shell.execute_reply.started":"2023-12-28T17:12:23.569122Z","shell.execute_reply":"2023-12-28T17:23:25.531303Z"},"trusted":true},"execution_count":50,"outputs":[{"output_type":"display_data","data":{"text/plain":"  0%|          | 0/125 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"cc14746c123443b19f0b45e73d7b938c"}},"metadata":{}},{"name":"stdout","text":"CPU times: user 11min, sys: 894 ms, total: 11min 1s\nWall time: 11min 1s\n","output_type":"stream"}]},{"cell_type":"code","source":"print(max_a, max_b)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:23:29.109777Z","iopub.execute_input":"2023-12-28T17:23:29.110302Z","iopub.status.idle":"2023-12-28T17:23:29.118405Z","shell.execute_reply.started":"2023-12-28T17:23:29.110264Z","shell.execute_reply":"2023-12-28T17:23:29.116763Z"},"trusted":true},"execution_count":51,"outputs":[{"name":"stdout","text":"1131410 471938\n","output_type":"stream"}]},{"cell_type":"code","source":"print(max_sim)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:23:30.323286Z","iopub.execute_input":"2023-12-28T17:23:30.323773Z","iopub.status.idle":"2023-12-28T17:23:30.331140Z","shell.execute_reply.started":"2023-12-28T17:23:30.323737Z","shell.execute_reply":"2023-12-28T17:23:30.329344Z"},"trusted":true},"execution_count":52,"outputs":[{"name":"stdout","text":"0.9898072551008147\n","output_type":"stream"}]},{"cell_type":"code","source":"df.iloc[[max_a, max_b]][[\"publish_date\", \"headline_text\"]]","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:23:31.731750Z","iopub.execute_input":"2023-12-28T17:23:31.732290Z","iopub.status.idle":"2023-12-28T17:23:31.749302Z","shell.execute_reply.started":"2023-12-28T17:23:31.732254Z","shell.execute_reply":"2023-12-28T17:23:31.747657Z"},"trusted":true},"execution_count":53,"outputs":[{"execution_count":53,"output_type":"execute_result","data":{"text/plain":"         publish_date       headline_text\n1131410      20180620   money money money\n471938       20090630  money in money out","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>publish_date</th>\n      <th>headline_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1131410</th>\n      <td>20180620</td>\n      <td>money money money</td>\n    </tr>\n    <tr>\n      <th>471938</th>\n      <td>20090630</td>\n      <td>money in money out</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Finding Related Words\n\nWords should be more related if they appear together in the corpus. ","metadata":{}},{"cell_type":"code","source":"tfidf_word = TfidfVectorizer(stop_words=list(stopwords), min_df=1000)       # Words should appear more\ndt_word = tfidf_word.fit_transform(df[\"headline_text\"])","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:07:55.430645Z","iopub.execute_input":"2023-12-28T17:07:55.431260Z","iopub.status.idle":"2023-12-28T17:08:14.759406Z","shell.execute_reply.started":"2023-12-28T17:07:55.431201Z","shell.execute_reply":"2023-12-28T17:08:14.757959Z"},"trusted":true},"execution_count":45,"outputs":[]},{"cell_type":"code","source":"r = cosine_similarity(dt_word.T, dt_word.T)\nnp.fill_diagonal(r, 0)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:08:14.761763Z","iopub.execute_input":"2023-12-28T17:08:14.762160Z","iopub.status.idle":"2023-12-28T17:08:16.177343Z","shell.execute_reply.started":"2023-12-28T17:08:14.762119Z","shell.execute_reply":"2023-12-28T17:08:16.175614Z"},"trusted":true},"execution_count":46,"outputs":[]},{"cell_type":"code","source":"voc = tfidf_word.get_feature_names_out()\nsize = r.shape[0] # quadratic\n\nfor index in np.argsort(r.flatten())[::-1][0:20]:\n    a = int(index / size)\n    b = index % size\n    if a > b:  # avoid repetitions\n        print('\"%s\" related to \"%s\"' % (voc[a], voc[b]))","metadata":{"execution":{"iopub.status.busy":"2023-12-28T17:08:24.715738Z","iopub.execute_input":"2023-12-28T17:08:24.717694Z","iopub.status.idle":"2023-12-28T17:08:24.976433Z","shell.execute_reply.started":"2023-12-28T17:08:24.717622Z","shell.execute_reply":"2023-12-28T17:08:24.975012Z"},"trusted":true},"execution_count":47,"outputs":[{"name":"stdout","text":"\"kong\" related to \"hong\"\n\"sri\" related to \"lanka\"\n\"covid\" related to \"19\"\n\"seekers\" related to \"asylum\"\n\"springs\" related to \"alice\"\n\"trump\" related to \"donald\"\n\"hour\" related to \"country\"\n\"pleads\" related to \"guilty\"\n\"hill\" related to \"broken\"\n\"vs\" related to \"summary\"\n","output_type":"stream"}]}]}