EDA and Pre-Processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "041a930c",
   "metadata": {},
   "source": [
    "TO DO:\n",
    "* DONE: Case folding\n",
    "* DONE: Stopwords, punctuation\n",
    "* DONE: Stemming\n",
    "4. Lemmatization (is this desirable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9d52ef8d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/maggiewu/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /Users/maggiewu/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import tweepy as tw\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.probability import FreqDist\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "nltk.download('wordnet')\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize import TweetTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fcb5b721",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 72388 entries, 0 to 72387\n",
      "Data columns (total 5 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   tweet_id   72388 non-null  int64 \n",
      " 1   text       72388 non-null  object\n",
      " 2   num_likes  72388 non-null  int64 \n",
      " 3   date       72388 non-null  object\n",
      " 4   author     72388 non-null  object\n",
      "dtypes: int64(2), object(3)\n",
      "memory usage: 2.8+ MB\n"
     ]
    }
   ],
   "source": [
    "file_path = \"hydrated.csv\"\n",
    "colnames = ['tweet_id', 'text', 'num_likes', 'date', 'author']\n",
    "hydrated_data = pd.read_csv(file_path, names = colnames)\n",
    "hydrated_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cea3c00d",
   "metadata": {},
   "outputs": [],
   "source": [
    "og_data = pd.read_csv(\"all_tweet_ids.csv\")\n",
    "\n",
    "labels = (og_data.loc[(og_data.Label == 'Hate') | (og_data.Label == 'Counterhate')]\\\n",
    "                     .rename(columns = {\"Tweet ID\" : \"tweet_id\"})\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "53a28f2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#join label information to hydrated data\n",
    "hydrated_data1 = pd.merge(hydrated_data, labels, how = 'left', on = 'tweet_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3c78074f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>text</th>\n",
       "      <th>num_likes</th>\n",
       "      <th>date</th>\n",
       "      <th>author</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1225778870093262849</td>\n",
       "      <td>@realDonaldTrump China has been trying to kill...</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-02-07 13:50:31</td>\n",
       "      <td>GailRusso19</td>\n",
       "      <td>Hate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1242461158654619649</td>\n",
       "      <td>Ignored warnings, shielded China: How WHO coll...</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-03-24 14:39:58</td>\n",
       "      <td>apivatkumara</td>\n",
       "      <td>Hate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1239916262479638528</td>\n",
       "      <td>@RyanAFournier @realDonaldTrump It’s not the C...</td>\n",
       "      <td>4</td>\n",
       "      <td>2020-03-17 14:07:28</td>\n",
       "      <td>cameronssmith__</td>\n",
       "      <td>Counterhate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1246361730873085953</td>\n",
       "      <td>@Grammarly I though REAL intellectuals develop...</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-04 08:59:27</td>\n",
       "      <td>Uyghur_Oghly</td>\n",
       "      <td>Hate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1241151165305364480</td>\n",
       "      <td>Since Wuhan virus is offensive perhaps we shou...</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-03-20 23:54:31</td>\n",
       "      <td>BBoveington</td>\n",
       "      <td>Hate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id                                               text  \\\n",
       "0  1225778870093262849  @realDonaldTrump China has been trying to kill...   \n",
       "1  1242461158654619649  Ignored warnings, shielded China: How WHO coll...   \n",
       "2  1239916262479638528  @RyanAFournier @realDonaldTrump It’s not the C...   \n",
       "3  1246361730873085953  @Grammarly I though REAL intellectuals develop...   \n",
       "4  1241151165305364480  Since Wuhan virus is offensive perhaps we shou...   \n",
       "\n",
       "   num_likes                 date           author        Label  \n",
       "0          0  2020-02-07 13:50:31      GailRusso19         Hate  \n",
       "1          0  2020-03-24 14:39:58     apivatkumara         Hate  \n",
       "2          4  2020-03-17 14:07:28  cameronssmith__  Counterhate  \n",
       "3          0  2020-04-04 08:59:27     Uyghur_Oghly         Hate  \n",
       "4          0  2020-03-20 23:54:31      BBoveington         Hate  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hydrated_data1.head()\n",
    "#hydrated_data1.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "591c0bb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>num_likes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7.238800e+04</td>\n",
       "      <td>72388.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.238868e+18</td>\n",
       "      <td>12.288542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.293131e+15</td>\n",
       "      <td>393.350479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.217413e+18</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.233764e+18</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.240593e+18</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.245013e+18</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.251299e+18</td>\n",
       "      <td>50632.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           tweet_id     num_likes\n",
       "count  7.238800e+04  72388.000000\n",
       "mean   1.238868e+18     12.288542\n",
       "std    8.293131e+15    393.350479\n",
       "min    1.217413e+18      0.000000\n",
       "25%    1.233764e+18      0.000000\n",
       "50%    1.240593e+18      0.000000\n",
       "75%    1.245013e+18      1.000000\n",
       "max    1.251299e+18  50632.000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hydrated_data1.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70dcb922",
   "metadata": {},
   "source": [
    "Out of the original 1,091,402 tweets, 10% were randomly sampled and hydrated. Out of the 109140 hydrated tweet ids, 66.6% were successfully retained (possible data loss is due to deletions). This leaves a corpus of 72,388 tweets.\n",
    "\n",
    "This data set includes the tweet text corpus (**text**) and other useful metadata: \n",
    "* **num_likes** the number of likes the tweet received and the tweet's author\n",
    "* **date** the date the tweet was posted\n",
    "* **author** the user who published the tweet\n",
    "* **tweet_id** ids for bookkeeping purposes\n",
    "* **Label** classification of the tweet as hate or counterhate\n",
    "\n",
    "There are additional metadata that are not included in the data frame that are saved in a list in my environment, such as the number of retweets, geolocation, etc. If such information is desirable, it can be added to this analysis."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60375891",
   "metadata": {},
   "source": [
    "** NLTK\n",
    "https://necromuralist.github.io/Neurotic-Networking/posts/nlp/01-twitter-preprocessing-with-nltk/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1309dcb1",
   "metadata": {},
   "source": [
    "### Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b925cdac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "@realDonaldTrump China has been trying to kill us for years.  Their virus is out of control.  They don’t have a hand on it.  We do. The warm weather will make it worse.  You are really stupid......\n",
      "Ignored warnings, shielded China: How WHO colluded with Chinese Communist Party to mislead the world about Coronavirus\n",
      "https://t.co/UE7pvywkGa #WHO #COVID19outbreak #CPC #WuhanVirus #ChinaLiedPeopleDied\n",
      "@RyanAFournier @realDonaldTrump It’s not the Chinese Virus...\n",
      "@Grammarly I though REAL intellectuals developed this app...\n",
      "hmmm,second though!..\n",
      "\n",
      "Did you know JUST the killed #Uyghurs by #China in 4 years are more than [3 millions]—#China targeted #Uyghurs who has no one oversees first to keep it secret!\n",
      "&amp; this virus originated from #Wuhan! https://t.co/J10KifoM37\n",
      "Since Wuhan virus is offensive perhaps we should call it Bat eating Commie bastard virus\n",
      "And this is why I refer to Covid19 as Wuhan virus. Pin it on America without any proof is the typical Red Chinese play https://t.co/lJqvV8c\n"
     ]
    }
   ],
   "source": [
    "s = hydrated_data1['text']\n",
    "corpus = s.str.cat(sep = '\\n', na_rep = '?')\n",
    "type(corpus)\n",
    "print(corpus[0:1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "624a320b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "#remove hyperlinks\n",
    "corpus = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "101546a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10964772"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7be0c3db",
   "metadata": {},
   "outputs": [],
   "source": [
    "import emoji\n",
    "\n",
    "corpus1 = emoji.get_emoji_regexp().sub(\"\", corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44973597",
   "metadata": {},
   "source": [
    "### Tokenizing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ebace81a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tw_tokenizer = TweetTokenizer()\n",
    "tokens = tw_tokenizer.tokenize(corpus1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76495e3d",
   "metadata": {},
   "source": [
    "### Case Folding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6b9db741",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74956"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normalized_tokens = [x.lower() for x in tokens]\n",
    "uniq=len(set(normalized_tokens))\n",
    "total=len(normalized_tokens)\n",
    "uniq/total\n",
    "uniq"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd1760e6",
   "metadata": {},
   "source": [
    "### Lexical Diversity\n",
    "After normalizing casing, the ratio of unique words to all words is 0.0377. There are 74,956 unique words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "62bd633f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# tw_tokenizer = TweetTokenizer()\n",
    "\n",
    "# def helper_tokenizer(x):\n",
    "#     return tw_tokenizer.tokenize(x)\n",
    "    \n",
    "# tokenized_list = hydrated_data.apply(lambda x: helper_tokenizer(x['text']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1adf9111",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('the', 84021),\n",
       " ('.', 82853),\n",
       " (',', 47225),\n",
       " ('to', 41805),\n",
       " ('china', 40611),\n",
       " ('is', 33789),\n",
       " ('virus', 33291),\n",
       " ('and', 31609),\n",
       " ('chinese', 31058),\n",
       " ('of', 29161),\n",
       " ('it', 27723),\n",
       " ('a', 25008),\n",
       " ('in', 23828),\n",
       " ('!', 21590),\n",
       " ('’', 17948),\n",
       " ('this', 16747),\n",
       " ('that', 16141),\n",
       " ('you', 15909),\n",
       " ('for', 14879),\n",
       " ('not', 14123)]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dist=FreqDist(normalized_tokens) #makes the text into a dictionary with key as words and value as frequency\n",
    "dist.most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b998a322",
   "metadata": {},
   "source": [
    "Clearly, we need to remove some stop words and punctuation!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e915d3e8",
   "metadata": {},
   "source": [
    "### Removing Undesirable Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d4d92d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/maggiewu/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "94597c65",
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords_list = stopwords.words('english')\n",
    "#appending some extra goodies to the stop word lexicon\n",
    "extra_stopwords = ['like', '19']\n",
    "stopwords_list.extend(extra_stopwords)\n",
    "\n",
    "#remove some extra punctuation\n",
    "extra_punctuation = ['…','...', '. . .', '’','“']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b1efab3d",
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "normalized_tokens = list(filter(lambda token: token not in string.punctuation, normalized_tokens))\n",
    "\n",
    "\n",
    "tweets_clean = []\n",
    "\n",
    "for word in normalized_tokens: # Go through every word in your tokens list\n",
    "    if (word not in stopwords_list and  # remove stopwords\n",
    "        word not in extra_punctuation):  # remove punctuation\n",
    "        tweets_clean.append(word)\n",
    "\n",
    "#len(tweets_clean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "616152b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('china', 40611),\n",
       " ('virus', 33291),\n",
       " ('chinese', 31058),\n",
       " ('world', 10748),\n",
       " ('people', 10607),\n",
       " ('wuhan', 8359),\n",
       " ('coronavirus', 7078),\n",
       " ('us', 5398),\n",
       " ('@realdonaldtrump', 5343),\n",
       " ('#chinesevirus', 5267),\n",
       " ('#chinaliedpeopledied', 4770),\n",
       " ('ccp', 4437),\n",
       " ('stop', 4221),\n",
       " ('communist', 4173),\n",
       " ('covid', 4145),\n",
       " ('racist', 3881),\n",
       " ('#china', 3831),\n",
       " ('calling', 3753),\n",
       " ('#wuhanvirus', 3579),\n",
       " ('#coronavirus', 3494)]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tweets_clean)\n",
    "dist=FreqDist(tweets_clean) #makes the text into a dictionary with key as words and value as frequency\n",
    "dist.most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70d8606c",
   "metadata": {},
   "source": [
    "Above is a list of the 20 most common words in the corpus (excluding stop words). Question: should I remove any extra words?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "979e01c6",
   "metadata": {},
   "source": [
    "### Lemmatization (Is this desirable?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86034cb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # import these modules\n",
    "# from nltk.stem import WordNetLemmatizer\n",
    "  \n",
    "# lemmatizer = WordNetLemmatizer()\n",
    "  \n",
    "# print(\"rocks :\", lemmatizer.lemmatize(\"rocks\"))\n",
    "# print(\"corpora :\", lemmatizer.lemmatize(\"corpora\"))\n",
    "  \n",
    "# # a denotes adjective in \"pos\"\n",
    "# print(\"better :\", lemmatizer.lemmatize(\"better\", pos =\"a\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5b1e351",
   "metadata": {},
   "source": [
    "### Stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4e5442ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import SnowballStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6c82a9c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate stemming class\n",
    "stemmer = PorterStemmer() \n",
    "\n",
    "# Create an empty list to store the stems\n",
    "porter_stemmed = [] \n",
    "\n",
    "for word in tweets_clean:\n",
    "    stem_word = stemmer.stem(word)  # stemming word\n",
    "    porter_stemmed.append(stem_word)  # append to the list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0f30494f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('china', 40658),\n",
       " ('viru', 33292),\n",
       " ('chines', 31071),\n",
       " ('world', 10809),\n",
       " ('peopl', 10734),\n",
       " ('call', 9000),\n",
       " ('wuhan', 8359),\n",
       " ('coronaviru', 7080),\n",
       " ('us', 5398),\n",
       " ('@realdonaldtrump', 5343),\n",
       " ('#chineseviru', 5268),\n",
       " ('#chinaliedpeopledi', 5265),\n",
       " ('countri', 4912),\n",
       " ('stop', 4608),\n",
       " ('spread', 4560),\n",
       " ('communist', 4504),\n",
       " ('ccp', 4445),\n",
       " ('fuck', 4257),\n",
       " ('covid', 4146),\n",
       " ('racist', 4089)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(porter_stemmed)\n",
    "dist=FreqDist(porter_stemmed) #makes the text into a dictionary with key as words and value as frequency\n",
    "dist.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5e6e879d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate stemming class\n",
    "snow = SnowballStemmer(\"english\")\n",
    "\n",
    "# Create an empty list to store the stems\n",
    "snowball_stemmed = [] \n",
    "\n",
    "for word in tweets_clean:\n",
    "    stem_word = snow.stem(word)  # stemming word\n",
    "    snowball_stemmed.append(stem_word)  # append to the list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4b50bb3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('china', 42562),\n",
       " ('virus', 33909),\n",
       " ('chines', 31085),\n",
       " ('world', 11017),\n",
       " ('peopl', 10926),\n",
       " ('call', 9000),\n",
       " ('wuhan', 8398),\n",
       " ('coronavirus', 7117),\n",
       " ('us', 5405),\n",
       " ('@realdonaldtrump', 5343),\n",
       " ('#chinesevirus', 5273),\n",
       " ('#chinaliedpeopledi', 5265),\n",
       " ('countri', 4956),\n",
       " ('stop', 4608),\n",
       " ('ccp', 4561),\n",
       " ('spread', 4560),\n",
       " ('communist', 4507),\n",
       " ('fuck', 4261),\n",
       " ('covid', 4146),\n",
       " ('racist', 4089)]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(snowball_stemmed)\n",
    "dist=FreqDist(snowball_stemmed) #makes the text into a dictionary with key as words and value as frequency\n",
    "dist.most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7e0f243",
   "metadata": {},
   "source": [
    "### Sentiment Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "26463be7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     /Users/maggiewu/nltk_data...\n"
     ]
    }
   ],
   "source": [
    "nltk.download('vader_lexicon')\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "fbb31c65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos: 0.088, "
     ]
    }
   ],
   "source": [
    "sia = SentimentIntensityAnalyzer()\n",
    "#tweets = hydrated_data1['text']\n",
    "#for t in tweets:\n",
    "ss = sia.polarity_scores(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7992a5f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "UsageError: Unknown variable 'ss'\n"
     ]
    }
   ],
   "source": [
    "%store ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "17fbaff1",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'dict' object has no attribute 'top_n'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-92-9931b0e964cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtop_n\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'dict' object has no attribute 'top_n'"
     ]
    }
   ],
   "source": [
    "ss.top_n()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "1123f9a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'neg': 0.183, 'neu': 0.729, 'pos': 0.088, 'compound': -1.0}"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ss"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83dc12ca",
   "metadata": {},
   "source": [
    "how do we label?\n",
    "parsing for parts of speech to pull out characters\n",
    "supervised approach where we manually label a training set and classify by narrative\n",
    "keyword based approach\n",
    "BERT tweet (ROBERTA) for clustering\n",
    "https://huggingface.co/transformers/model_doc/bertweet.html\n",
    "manifestation of the tweets\n",
    "measuring impact via engagement statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0835713",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "986dea5987cb21865cf36c17290b93693e3f7f623729ad5c9a052aa91ddaf825"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}