TweetsAnalyzer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "if 'SPARK_HOME' not in os.environ:\n",
    "    os.environ['SPARK_HOME'] = \"/usr/bin/spark-3.0.0-preview2-bin-hadoop2.7\"\n",
    "if 'PYSPARK_PYTHON' not in os.environ:\n",
    "    os.environ['PYSPARK_PYTHON'] = \"/usr/bin/python3\"\n",
    "    \n",
    "    \n",
    "SPARK_HOME = os.environ['SPARK_HOME']\n",
    "\n",
    "sys.path.insert(0,  os.path.join(SPARK_HOME, 'python'))\n",
    "sys.path.insert(0, os.path.join(SPARK_HOME, 'python', 'lib'))\n",
    "sys.path.insert(0, os.path.join(SPARK_HOME, 'python', 'lib', 'pyspark.zip'))\n",
    "sys.path.insert(0, os.path.join(SPARK_HOME, 'python', 'lib', 'py4j-0.10.8.1-src.zip'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import isnan, when\n",
    "from pyspark.ml.feature import HashingTF, IDF, Tokenizer\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "\n",
    "pd.set_option('display.max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['sentiment', 'id', 'date', 'query_string', 'user', 'text']\n",
    "tweets = pd.read_csv(\"./training.1600000.processed.noemoticon.csv\", \n",
    "                 header=None,\n",
    "                 names=cols,\n",
    "                 encoding=\"ISO-8859-1\"\n",
    "                )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>id</th>\n",
       "      <th>date</th>\n",
       "      <th>query_string</th>\n",
       "      <th>user</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment          id                          date query_string  \\\n",
       "0  0          1467810369  Mon Apr 06 22:19:45 PDT 2009  NO_QUERY      \n",
       "1  0          1467810672  Mon Apr 06 22:19:49 PDT 2009  NO_QUERY      \n",
       "2  0          1467810917  Mon Apr 06 22:19:53 PDT 2009  NO_QUERY      \n",
       "3  0          1467811184  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY      \n",
       "4  0          1467811193  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY      \n",
       "\n",
       "              user  \\\n",
       "0  _TheSpecialOne_   \n",
       "1  scotthamilton     \n",
       "2  mattycus          \n",
       "3  ElleCTF           \n",
       "4  Karoli            \n",
       "\n",
       "                                                                                                                  text  \n",
       "0  @switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D  \n",
       "1  is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!      \n",
       "2  @Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds                            \n",
       "3  my whole body feels itchy and like its on fire                                                                       \n",
       "4  @nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there.       "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    800000\n",
       "0    800000\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets.drop(columns=['id', 'date', 'query_string', 'user'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment  \\\n",
       "0  0           \n",
       "1  0           \n",
       "2  0           \n",
       "3  0           \n",
       "4  0           \n",
       "\n",
       "                                                                                                                  text  \n",
       "0  @switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D  \n",
       "1  is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!      \n",
       "2  @Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds                            \n",
       "3  my whole body feels itchy and like its on fire                                                                       \n",
       "4  @nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there.       "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets[tweets['sentiment'] == 0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>800000</td>\n",
       "      <td>4</td>\n",
       "      <td>I LOVE @Health4UandPets u guys r the best!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800001</td>\n",
       "      <td>4</td>\n",
       "      <td>im meeting up with one of my besties tonight! Cant wait!!  - GIRL TALK!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800002</td>\n",
       "      <td>4</td>\n",
       "      <td>@DaRealSunisaKim Thanks for the Twitter add, Sunisa! I got to meet you once at a HIN show here in the DC area and you were a sweetheart.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800003</td>\n",
       "      <td>4</td>\n",
       "      <td>Being sick can be really cheap when it hurts too much to eat real food  Plus, your friends make you soup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800004</td>\n",
       "      <td>4</td>\n",
       "      <td>@LovesBrooklyn2 he has that effect on everyone</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        sentiment  \\\n",
       "800000  4           \n",
       "800001  4           \n",
       "800002  4           \n",
       "800003  4           \n",
       "800004  4           \n",
       "\n",
       "                                                                                                                                             text  \n",
       "800000  I LOVE @Health4UandPets u guys r the best!!                                                                                                \n",
       "800001  im meeting up with one of my besties tonight! Cant wait!!  - GIRL TALK!!                                                                   \n",
       "800002  @DaRealSunisaKim Thanks for the Twitter add, Sunisa! I got to meet you once at a HIN show here in the DC area and you were a sweetheart.   \n",
       "800003  Being sick can be really cheap when it hurts too much to eat real food  Plus, your friends make you soup                                   \n",
       "800004  @LovesBrooklyn2 he has that effect on everyone                                                                                             "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets[tweets['sentiment'] == 4].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['pre_clean_len'] = [len(t) for t in tweets['text']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT8AAAE/CAYAAAAwpsSrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcB0lEQVR4nO3df3RX9Z3n8eeLCEkFFTMGlgGsThenUHYGuxnUlVEpp1Rs58DsTqu0Q21JpZy1WXt0VxT+UOcUju7647Q4bbSSaqeYyukPqxXXQUz1ZJnaxtahCOMptVqSQYlFBaMkIbz3j+8N/YYmJPl+E7+53NfjnO/JvZ/7650QXrmf+1MRgZlZ1owpdQFmZqXg8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5/1IulmSd8pdR2jnaTPSWoq0bbvl/SVUmz7ROLws1FB0k8kfeE93mbJAmyw0lBjWjn8TgCSTip1DWZp4/AbpSS9LOlGSTslvSHpW5IqkmmXSGqRtErSq8C3kvZPSHpe0puStkn6i+Os/0OStkjaL+k1Sav7me/8ZF1vSvpXSZfkTfu8pF2SDkp6SdIX86b11HidpH2S9kr6fD/bWAv8NXC3pLcl3S3pFknrk+ljJbVL+j/J+PskHZJUOYgaT5O0Idl+q6SvSCqTNBOoAy5ItvlmMv9lyc/8YDL//xzEPxeSPpj383xR0qfypt0v6R8lPZas91lJH8ibvjBZ5i1JX5f0tKQv9Fdj4vT+1meDFBH+jMIP8DKwA5gOVAL/D/hKMu0S4DBwG1AOvA84F9gHnAeUAVcm6yjvY92nAHuB64CKZPy8ZNrNwHeS4anA74HLyP2h/GgyXpVM/zjwAUDAxcA7wIePqfEfgLHJOt4BTu/n+/0J8IW88Y8Av0qG/wvwG+DZvGn/OsgafwjcA4wHJgE/A76YTPsc0HRMHXuBv06GT+/5fvqo9+iyybr3AJ8HTkr+LV4HZiXT709qmptM3wh8N5l2BnAA+K/JtGuArp6fRT819rs+fwb/8Z7f6HZ3ROyJiP3AWmBp3rQjwE0R0RER7wIrgHsi4tmI6I6IB4AO4Pw+1vsJ4NWIuCMiDkXEwYh4to/5/h7YHBGbI+JIRGwBmskFDRHxWET8JnKeBv6Z3B5cjy7gHyKiKyI2A28Dfz7I7/1fgBmS/gS4CNgATJU0gVzQPj1QjZImJ7V+OSLaI2IfcBdwxXG22wXMknRqRLwREb8YRK2fAF6OiG9FxOGI+CXwfeCTefP8MCJ+FhGHyYXVnKT9MuCFiPhBMu1rwKuD2GZ/67NBcviNbnvyhl8B/jRvvC0iDuWNvx+4Lun6vZl0kaYfs0yP6eT2pAbyfuCTx6xzHjAFQNIiST9NunpvkvuPfEbe8r9P/nP2eAeYMIjtkgR6M7mgu4hc2G0DLqR3+B2vxveT2+vcmzftHnJ7gP35b8n38UrS/bxgEOW+HzjvmBo+A/yHvHnyAy3/5/Cn5P07R27XrmUQ2+xvfTZIPlA+uk3PGz4T+Pe88WMfx7MHWBsRawex3j0cf+8nf75/ioirjp0gqZzc3s1ngR9FRJekh8l1gQvR1+OFnibXxT0X+Hky/jFy3b1nBlHjFHJ7v2ccE8L9bjMifg4sljQW+BKwid7/Dn3ZAzwdER8dYL6+7AWm5dWs/PG+arTh4T2/0e1qSdOSA/trgIeOM+83gZWSzlPOeEkfl3RKH/P+GJgi6cuSyiWdIum8Pub7DvA3kj6WnCSoSE5kTAPGkTve2AYclrQIWFjE9/oa8GfHtD1NLlx3RkQnyXFB4LcR0TZQjRGxl1xX/A5Jp0oaI+kDki7O2+Y0SeMAJI2T9BlJp0VEF7ljcUcGUfuPgXMkLUtOzoyV9FfJCYuBPAb8J0lLlDtrfzW99xh71WjDx+E3uj1I7j/vS+S6qf1e2BoRzcBVwN3AG8BucgfL+5r3ILkTA39Drvv0a2B+H/PtARYDq8mF3B7gfwFjknX8D3J7Rm8AnwYeGfq3eNRXgb9T7sz215K2beRO5vTs5e0EDuWNH7fGZJbPkgvqnUmd3yPptgNPAS8Ar0p6PWlbBrws6QCwklz39biSn8VCcnvT/07uZ9pzMmqgZV8nd2zwf5M7iTGLXHe/4zg12jBQ7hCDjTaSXiZ3xu/JUtdi7x1JY8gd8/tMRDSWup4Tmff8zEos6bJPTI6jriZ33PSnJS7rhOfwMyu9C8gd1nid3KGIJcnZbhtB7vaaWSZ5z8/MMsnhZ2aZNCoucj7jjDPirLPOKnUZZnaCee65516PiKq+po2K8DvrrLNobm4udRlmdoKR9Ep/09ztNbNMcviZWSY5/Mwskxx+ZpZJDj8zyySHn5llksPPzDLJ4Wep0dDQwOzZsykrK2P27Nk0NDSUuiRLsVFxkbPZQBoaGlizZg0bNmxg3rx5NDU1UVNTA8DSpUsHWNrsj42Kp7pUV1eH7/Cw45k9ezbr169n/vw/PHC6sbGR2tpaduzYUcLKbDST9FxEVPc5zeFnaVBWVsahQ4cYO3bs0bauri4qKiro7u4uYWU2mh0v/HzMz1Jh5syZ3HLLLb2O+d1yyy3MnDmYdwSZ/TGHn6XC/Pnzue2221i+fDkHDx5k+fLl3Hbbbb26wWZD4fCzVGhsbGTVqlXU19dzyimnUF9fz6pVq2hs9Dt+rDA+5mep4GN+Vggf87PUmzlzJk1NTb3ampqafMzPCubws1RYs2YNNTU1NDY20tXVRWNjIzU1NaxZs6bUpVlKOfwsFZYuXcqMGTNYsGAB48aNY8GCBcyYMcMXOFvBHH6WCrW1tTz11FPcfvvttLe3c/vtt/PUU09RW1tb6tIspXzCw1KhoqKCdevWce211x5tu/POO1m9ejWHDh0qYWU2mvkOD0s9SbS3t3PyyScfbXvnnXcYP348o+F32EYnn+211CsvL6eurq5XW11dHeXl5SWqyNLOT3WxVLjqqqtYtWoVACtXrqSuro5Vq1axcuXKEldmaTVg+EmqAJ4BypP5vxcRN0m6H7gYeCuZ9XMR8bwkAV8FLgPeSdp/MRLFW3asX78egNWrV3PddddRXl7OypUrj7abDdWAx/ySMBsfEW9LGgs0AdcAK4EfR8T3jpn/MqCWXPidB3w1Is473jZ8zM/MRkJRx/wi5+1kdGzyOV5iLga+nSz3U2CipClDLdrsWH6Ssw2nQZ3wkFQm6XlgH7AlIp5NJq2VtF3SXZJ6jjxPBfbkLd6StB27zhWSmiU1t7W1FfEtWBb0PMl5/fr1HDp0iPXr17NmzRoHoBVsUOEXEd0RMQeYBsyVNBu4Efgg8FdAJbBqKBuOiHsjojoiqquqqoZYtmXN2rVr2bBhA/Pnz2fs2LHMnz+fDRs2sHbt2lKXZik1pEtdIuJNoBG4NCL2Jl3bDuBbwNxktlZget5i05I2s4Lt2rWLefPm9WqbN28eu3btKlFFlnYDhp+kKkkTk+H3AR8F/q3nOF5yQmQJ0PMihUeAzyrnfOCtiNg7ItVbZvipLjbcBrPnNwVolLQd+Dm5Y34/BjZK+hXwK+AM4CvJ/JuBl4DdwDeB/z7sVVvm+KkuNtwGvM4vIrYD5/bR/pF+5g/g6uJLM/uDnqe31NbWsmvXLmbOnMnatWv9VBcrmO/tNbMTlu/tNTM7hsPPzDLJ4WdmmeTwM7NMcviZWSY5/Cw1/GADG05+mKmlQs+DDTZs2MC8efNoamqipqYGwNf6WUF8nZ+lwuzZs1myZAkPP/zw0Yuce8Z37Ngx8Aosk453nZ/3/CwVdu7cSXt7O/X19Uf3/JYvX84rr7xS6tIspRx+lgrjxo3jwgsv7HV724UXXsjevX5mhhXGJzwsFTo6OnjooYdYvnw5Bw8eZPny5Tz00EN0dHSUujRLKYefpUJ5eTmXX3459fX1nHLKKdTX13P55Zf71ZVWMIefpUJnZyfbtm3r9Rj7bdu20dnZWerSLKV8zM9SYdasWSxZsqTXMb9Pf/rTPPzww6UuzVLKe36WCmvWrOHBBx/stef34IMP+mGmVjCHn6XC0qVLmTFjBgsWLGDcuHEsWLCAGTNm+AJnK5jDz1KhtraWJ598kkmTJgEwadIknnzySWpra0tcmaWVw89Soa6ujtNOO42GhgY6OztpaGjgtNNOo66urtSlWUo5/CwVDh8+zMaNG3u9t3fjxo0cPny41KVZSjn8LDWOvYfX9/RaMXypi6VCZWUlN954I2VlZaxcuZK6ujpuvPFGKisrS12apZT3/CwV7r77bk4++WRuuOEGxo8fzw033MDJJ5/M3XffXerSLKUcfpYKS5cu5Z577uGcc85hzJgxnHPOOdxzzz2+1MUKNuDz/CRVAM8A5eS6yd+LiJsknQ18F/gT4DlgWUR0SioHvg38Z+D3wOUR8fLxtuHn+ZnZSCj2vb0dwEci4i+BOcClks4HbgPuioj/CLwB1CTz1wBvJO13JfOZFc2PsbfhNGD4Rc7byejY5BPAR4DvJe0PAEuS4cXJOMn0BZI0bBVbJjU0NHDNNdfQ3t5ORNDe3s4111zjALSCDeqYn6QySc8D+4AtwG+ANyOi5yKrFmBqMjwV2AOQTH+LXNfYrGDXX389ZWVl1NfX09HRQX19PWVlZVx//fWlLs1SalDhFxHdETEHmAbMBT5Y7IYlrZDULKm5ra2t2NXZCa6lpYW5c+eyaNEixo0bx6JFi5g7dy4tLS2lLs1SakhneyPiTaARuACYKKnnOsFpQGsy3ApMB0imn0buxMex67o3IqojorqqqqrA8i1LHn30USZOnAjAxIkTefTRR0tckaXZgOEnqUrSxGT4fcBHgV3kQvDvktmuBH6UDD+SjJNMfypGwyvi7IRw/fXX097e7u6uFW0wl7r8BbkTGGXkwnJTRPyDpD8jd6lLJfBL4O8joiO5NOafgHOB/cAVEfHS8bbhS11sIJI49dRTqays5He/+x1nnnkm+/fv58CBA/hvq/WnqFdXRsR2ckF2bPtL5I7/Hdt+CPhkAXWaHdfHP/5xtm/fDsD48eO54IILfLbXCuY7PCwVKisr2bRpU6+3t23atMn39lrBHH6WCr6314abw89Swff22nAb8ITHe8EnPMxsJBR7b6/ZqFBbW0tFRQWSqKio8Ps7rCgOP0uF2tpa6urqWLduHe3t7axbt466ujoHoBXM3V5LhYqKCtatW8e11157tO3OO+9k9erVHDp0qISV2Wjmbq+lXkdHBy+++GKvbu+LL75IR0dHqUuzlHL4WSqMGTOG++67r1e397777mPMGP8KW2Hc7bVUOOmkk+ju7mby5Mns27ePSZMm8dprr1FWVubXV1q/3O211Ovu7mbChAns37+fiGD//v1MmDCB7u7uUpdmKeXws1SQxLJly+js7CQi6OzsZNmyZfgh4VYod3stFSQhiTFjxtDd3U1ZWRlHjhwhIvxUF+uXu72Wev09wMAPNrBCOfwsFQ4cOMDEiRPZsmULnZ2dbNmyhYkTJ3LgwIFSl2Yp5fCzVDh8+DB33HHH0VvcamtrueOOO3ym1wrm8LNUKC8vZ+vWrb3atm7dSnl5eYkqsrRz+FkqXHzxxWzcuJGLLrqI/fv3c9FFF7Fx40YuvvjiUpdmKeXws1RobW1lyZIl1NfXM3HiROrr61myZAmtra0DL2zWhwHf4WE2GuzatYtf/vKXjB079mhbV1cXFRUVJazK0sx7fpYKM2fOpKmpqVdbU1MTM2fOLFFFlnYOP0uFNWvWUFNTQ2NjI11dXTQ2NlJTU8OaNWtKXZqllLu9lgpLly5l27ZtLFq0iI6ODsrLy7nqqqv8Dg8rmPf8LBUaGhp47LHHePzxx+ns7OTxxx/nscce83t7rWC+t9dSYfbs2SxZsoSHH36YXbt2MXPmzKPjO3bsKHV5NkoVdW+vpOmSGiXtlPSCpGuS9psltUp6PvlclrfMjZJ2S3pR0seG71uxrNq5cycbN25k/fr1HDp0iPXr17Nx40Z27txZ6tIspQbT7T0MXBcRs4DzgaslzUqm3RURc5LPZoBk2hXAh4BLga9LKhuB2i1Dxo0bR21tLfPnz2fs2LHMnz+f2tpaxo0bV+rSLKUGDL+I2BsRv0iGDwK7gKnHWWQx8N2I6IiI3wK7gbnDUaxlV2dnJ7feeitnn302Y8aM4eyzz+bWW2+ls7Oz1KVZSg3phIeks4BzgWeTpi9J2i6pXtLpSdtUYE/eYi0cPyzNBjR16lS6uroAjj7AtKuri6lT/atlhRl0+EmaAHwf+HJEHAC+AXwAmAPsBe4YyoYlrZDULKm5ra1tKItaRnV0dNDa2sqRI0dobW31m9usKIMKP0ljyQXfxoj4AUBEvBYR3RFxBPgmf+jatgLT8xaflrT1EhH3RkR1RFRXVVUV8z1YBrS0tPDuu+8e3fvr6uri3XffpaWlpcSVWVoN5myvgA3Aroi4M699St5sfwv0XG/wCHCFpHJJZwMzgJ8NX8mWZWVlZb2+mhVqMHd4XAgsA34l6fmkbTWwVNIcIICXgS8CRMQLkjYBO8mdKb46IvyKLRsWPW9r81vbrFgDhl9ENAF9vSJr83GWWQusLaIuM7MR5dvbLFUmTJjQ66tZoRx+lipvv/12r69mhXL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyacDwkzRdUqOknZJekHRN0l4paYukXydfT0/aJelrknZL2i7pwyP9TZiZDdVg9vwOA9dFxCzgfOBqSbOAG4CtETED2JqMAywCZiSfFcA3hr1qM7MiDRh+EbE3In6RDB8EdgFTgcXAA8lsDwBLkuHFwLcj56fARElThr1yM7MiDOmYn6SzgHOBZ4HJEbE3mfQqMDkZngrsyVusJWkzMxs1Bh1+kiYA3we+HBEH8qdFRAAxlA1LWiGpWVJzW1vbUBY1MyvaoMJP0lhywbcxIn6QNL/W051Nvu5L2luB6XmLT0vaeomIeyOiOiKqq6qqCq3fzKwggznbK2ADsCsi7syb9AhwZTJ8JfCjvPbPJmd9zwfeyusem5mNCicNYp4LgWXAryQ9n7StBm4FNkmqAV4BPpVM2wxcBuwG3gE+P6wVm5kNgwHDLyKaAPUzeUEf8wdwdZF1WcbkOhjDv2zu19Hsjw1mz89sxA0UUg44G26+vc1SYeHChUNqNxuIw89S4YknnmDhwoVH9wAlsXDhQp544okSV2Zp5W6vpUZP0EniyJEjJa7G0s57fmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpk0YPhJqpe0T9KOvLabJbVKej75XJY37UZJuyW9KOljI1W4mVkxBrPndz9waR/td0XEnOSzGUDSLOAK4EPJMl+XVDZcxZqZDZcBwy8ingH2D3J9i4HvRkRHRPwW2A3MLaI+M7MRUcwxvy9J2p50i09P2qYCe/LmaUnazMxGlULD7xvAB4A5wF7gjqGuQNIKSc2Smtva2gosw8ysMAWFX0S8FhHdEXEE+CZ/6Nq2AtPzZp2WtPW1jnsjojoiqquqqgopw8ysYAWFn6QpeaN/C/ScCX4EuEJSuaSzgRnAz4or0cxs+J000AySGoBLgDMktQA3AZdImgME8DLwRYCIeEHSJmAncBi4OiK6R6Z0M7PCKSJKXQPV1dXR3Nxc6jIsJSQxGn5vbfST9FxEVPc1zXd4mFkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmDRh+kuol7ZO0I6+tUtIWSb9Ovp6etEvS1yTtlrRd0odHsngzs0INZs/vfuDSY9puALZGxAxgazIOsAiYkXxWAN8YnjItrSorK5E0rB9g2NdZWVlZ4p+UvddOGmiGiHhG0lnHNC8GLkmGHwB+AqxK2r8dEQH8VNJESVMiYu9wFWzp8sYbb5D7dRjdekLVsqPQY36T8wLtVWByMjwV2JM3X0vS9kckrZDULKm5ra2twDLMzApT9AmPZC9vyH/aI+LeiKiOiOqqqqpiyzAzG5JCw+81SVMAkq/7kvZWYHrefNOSNjOzUaXQ8HsEuDIZvhL4UV77Z5OzvucDb/l4n5mNRgOe8JDUQO7kxhmSWoCbgFuBTZJqgFeATyWzbwYuA3YD7wCfH4GazcyKNpizvUv7mbSgj3kDuLrYoszMRprv8DCzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTHL4mVkmOfzMLJMcfmaWSQ4/M8skh5+ZZZLDz8wyacDn+ZkVI246FW4+rdRlDChuOrXUJdh7zOFnI0q3HEjNqyvj5lJXYe8ld3vNLJMcfmaWSQ4/M8skh5+ZZZLDz8wyyeFnZpnk8DOzTCrqOj9JLwMHgW7gcERUS6oEHgLOAl4GPhURbxRXppnZ8BqOPb/5ETEnIqqT8RuArRExA9iajJuZjSoj0e1dDDyQDD8ALBmBbZiZFaXY8AvgnyU9J2lF0jY5IvYmw68Ck4vchpnZsCv23t55EdEqaRKwRdK/5U+MiJDU542dSViuADjzzDOLLMPMbGiK2vOLiNbk6z7gh8Bc4DVJUwCSr/v6WfbeiKiOiOqqqqpiyjAzG7KCw0/SeEmn9AwDC4EdwCPAlclsVwI/KrZIM7PhVky3dzLwQ0k963kwIv6vpJ8DmyTVAK8Anyq+TDOz4VVw+EXES8Bf9tH+e2BBMUXZiSX5AzmqnX766aUuwd5jfpipjaiReJCppFQ8INVGN9/eZmaZ5PAzs0xy+JlZJjn8zCyTHH5mlkkOPzPLJIefmWWSw8/MMsnhZ2aZ5PAzs0xy+JlZJjn8zCyTHH5mlkkOPzPLJIefmWWSw8/MMsnhZ2aZ5PAzs0xy+JlZJjn8zCyTHH5mlkkOPzPLJIefmWWS39tro8JQX2w+2Pn9fl/rz4jt+Um6VNKLknZLumGktmMnhogYkY9Zf0Yk/CSVAf8ILAJmAUslzRqJbZmZFWKk9vzmArsj4qWI6AS+CyweoW2ZmQ3ZSIXfVGBP3nhL0naUpBWSmiU1t7W1jVAZZmZ9K9nZ3oi4NyKqI6K6qqqqVGWYWUaNVPi1AtPzxqclbWZmo8JIhd/PgRmSzpY0DrgCeGSEtmVmNmQjcp1fRByW9CXgCaAMqI+IF0ZiW2ZmhRixi5wjYjOweaTWb2ZWDN/eZmaZ5PAzs0xy+JlZJmk03P8oqQ14pdR1WGqcAbxe6iIsFd4fEX1eSDwqws9sKCQ1R0R1qeuwdHO318wyyeFnZpnk8LM0urfUBVj6+ZifmWWS9/zMLJMcfpYakuol7ZO0o9S1WPo5/CxN7gcuLXURdmJw+FlqRMQzwP5S12EnBoefmWWSw8/MMsnhZ2aZ5PAzs0xy+FlqSGoA/gX4c0ktkmpKXZOll+/wMLNM8p6fmWWSw8/MMsnhZ2aZ5PAzs0xy+JlZJjn8zCyTHH5mlkkOPzPLpP8PTOgG5NkdnrYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(5, 5))\n",
    "plt.boxplot(tweets['pre_clean_len'])\n",
    "plt.title('pre clean tweets length')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data cleaning\n",
    "\n",
    "def tweet_cleaner(text):\n",
    "    \n",
    "    # 1. first step is to decode html tags to general tags.\n",
    "    soup = BeautifulSoup(text, 'lxml')\n",
    "    souped = soup.get_text()\n",
    "    \n",
    "    # 2. remove @mention handles \n",
    "    handle_pat = r'@[A-Za-z0-9_]+'\n",
    "    han_rem = re.sub(handle_pat, '', souped)\n",
    "    \n",
    "    # 3. remove http url\n",
    "    url_pat = r'https?://[A-Za-z0-9./]+'\n",
    "    url_rem = re.sub(url_pat, '', han_rem)\n",
    "    \n",
    "    # 4. remove www url\n",
    "    www_pat = r'www.[^ ]+'\n",
    "    www_rem = re.sub(www_pat, '', url_rem)\n",
    "    \n",
    "    # 5. decode the text with 'utf-8-sig' , to remove utf-8 BOM sequence (\\xef\\xbf\\xbd).\n",
    "    try:\n",
    "        cleaned_text = www_rem.decode('utf-8-sig').replace(u'\\ufffd', '?')\n",
    "    except:\n",
    "        cleaned_text = www_rem\n",
    "        \n",
    "    # 6. handle negation words( can't -> can not)\n",
    "    neg_handeled = re.sub(r\"won't\", 'will not', cleaned_text)\n",
    "    neg_handeled = re.sub(r\"can't\", 'can not', neg_handeled)\n",
    "    neg_handeled = re.sub(r\"n't\", ' not', neg_handeled)\n",
    "        \n",
    "    # 7. remove special chars\n",
    "    text_only = re.sub('[^A-Za-z]', ' ', neg_handeled)\n",
    "    \n",
    "    # 8. covert text to lower case and remove unwanted space added by above step\n",
    "    low_text = text_only.lower()\n",
    "    final_text = low_text.split()\n",
    "    \n",
    "    return (' '.join(final_text))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed 100000 tweets\n",
      "processed 200000 tweets\n",
      "processed 300000 tweets\n",
      "processed 400000 tweets\n",
      "processed 500000 tweets\n",
      "processed 600000 tweets\n",
      "processed 700000 tweets\n",
      "processed 800000 tweets\n",
      "processed 900000 tweets\n",
      "processed 1000000 tweets\n",
      "processed 1100000 tweets\n",
      "processed 1200000 tweets\n",
      "processed 1300000 tweets\n",
      "processed 1400000 tweets\n",
      "processed 1500000 tweets\n"
     ]
    }
   ],
   "source": [
    "cleaned_tweets_text = []\n",
    "batch = 100000\n",
    "for i in range(len(tweets)):\n",
    "    if i > 0 and i % batch == 0 :\n",
    "        print(f'processed {i} tweets')\n",
    "    cleaned_tweets_text.append(tweet_cleaner(tweets['text'][i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>awww that s a bummer you shoulda got david carr of third day to do it d</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>is upset that he can not update his facebook by texting it and might cry as a result school today also blah</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>i dived many times for the ball managed to save the rest go out of bounds</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>no it s not behaving at all i m mad why am i here because i can not see you all over there</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>not the whole crew</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>need a hug</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>hey long time no see yes rains a bit only a bit lol i m fine thanks how s you</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>nope they did not have it</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>que me muera</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>spring break in plain city it s snowing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>i just re pierced my ears</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>i could not bear to watch it and i thought the ua loss was embarrassing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>it it counts idk why i did either you never talk to me anymore</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>i would ve been the first but i did not have a gun not really though zac snyder s just a doucheclown</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>i wish i got to watch it with you i miss you and how was the premiere</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>hollis death scene will hurt me severely to watch on film wry is directors cut not out now</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>about to file taxes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>ahh ive always wanted to see rent love the soundtrack</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>oh dear were you drinking out of the forgotten table drinks</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                           text  \\\n",
       "0   awww that s a bummer you shoulda got david carr of third day to do it d                                       \n",
       "1   is upset that he can not update his facebook by texting it and might cry as a result school today also blah   \n",
       "2   i dived many times for the ball managed to save the rest go out of bounds                                     \n",
       "3   my whole body feels itchy and like its on fire                                                                \n",
       "4   no it s not behaving at all i m mad why am i here because i can not see you all over there                    \n",
       "5   not the whole crew                                                                                            \n",
       "6   need a hug                                                                                                    \n",
       "7   hey long time no see yes rains a bit only a bit lol i m fine thanks how s you                                 \n",
       "8   nope they did not have it                                                                                     \n",
       "9   que me muera                                                                                                  \n",
       "10  spring break in plain city it s snowing                                                                       \n",
       "11  i just re pierced my ears                                                                                     \n",
       "12  i could not bear to watch it and i thought the ua loss was embarrassing                                       \n",
       "13  it it counts idk why i did either you never talk to me anymore                                                \n",
       "14  i would ve been the first but i did not have a gun not really though zac snyder s just a doucheclown          \n",
       "15  i wish i got to watch it with you i miss you and how was the premiere                                         \n",
       "16  hollis death scene will hurt me severely to watch on film wry is directors cut not out now                    \n",
       "17  about to file taxes                                                                                           \n",
       "18  ahh ive always wanted to see rent love the soundtrack                                                         \n",
       "19  oh dear were you drinking out of the forgotten table drinks                                                   \n",
       "\n",
       "    target  \n",
       "0   0       \n",
       "1   0       \n",
       "2   0       \n",
       "3   0       \n",
       "4   0       \n",
       "5   0       \n",
       "6   0       \n",
       "7   0       \n",
       "8   0       \n",
       "9   0       \n",
       "10  0       \n",
       "11  0       \n",
       "12  0       \n",
       "13  0       \n",
       "14  0       \n",
       "15  0       \n",
       "16  0       \n",
       "17  0       \n",
       "18  0       \n",
       "19  0       "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_tweets = pd.DataFrame(cleaned_tweets_text, columns=['text'])\n",
    "cleaned_tweets['target'] = tweets['sentiment']\n",
    "cleaned_tweets.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_tweets.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_tweets['length'] = cleaned_tweets['text'].str.split().apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>awww that s a bummer you shoulda got david carr of third day to do it d</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>is upset that he can not update his facebook by texting it and might cry as a result school today also blah</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>i dived many times for the ball managed to save the rest go out of bounds</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>no it s not behaving at all i m mad why am i here because i can not see you all over there</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                          text  \\\n",
       "0  awww that s a bummer you shoulda got david carr of third day to do it d                                       \n",
       "1  is upset that he can not update his facebook by texting it and might cry as a result school today also blah   \n",
       "2  i dived many times for the ball managed to save the rest go out of bounds                                     \n",
       "3  my whole body feels itchy and like its on fire                                                                \n",
       "4  no it s not behaving at all i m mad why am i here because i can not see you all over there                    \n",
       "\n",
       "   target  length  \n",
       "0  0       17      \n",
       "1  0       22      \n",
       "2  0       16      \n",
       "3  0       10      \n",
       "4  0       23      "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_tweets.to_csv('cleaned_tweets.csv', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession \\\n",
    "        .builder \\\n",
    "        .appName('TweetsAnalyzer') \\\n",
    "        .getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = spark.read.csv(\"cleaned_tweets.csv\", header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1600000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: string (nullable = true)\n",
      " |-- text: string (nullable = true)\n",
      " |-- target: string (nullable = true)\n",
      " |-- length: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tweets_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+--------------------+------+------+\n",
      "|_c0|                text|target|length|\n",
      "+---+--------------------+------+------+\n",
      "|  0|awww that s a bum...|     0|    17|\n",
      "|  1|is upset that he ...|     0|    22|\n",
      "|  2|i dived many time...|     0|    16|\n",
      "|  3|my whole body fee...|     0|    10|\n",
      "|  4|no it s not behav...|     0|    23|\n",
      "|  5|  not the whole crew|     0|     4|\n",
      "|  6|          need a hug|     0|     3|\n",
      "|  7|hey long time no ...|     0|    20|\n",
      "|  8|nope they did not...|     0|     6|\n",
      "|  9|        que me muera|     0|     3|\n",
      "| 10|spring break in p...|     0|     8|\n",
      "| 11|i just re pierced...|     0|     6|\n",
      "| 12|i could not bear ...|     0|    15|\n",
      "| 13|it it counts idk ...|     0|    14|\n",
      "| 14|i would ve been t...|     0|    22|\n",
      "| 15|i wish i got to w...|     0|    17|\n",
      "| 16|hollis death scen...|     0|    18|\n",
      "| 17| about to file taxes|     0|     4|\n",
      "| 18|ahh ive always wa...|     0|    10|\n",
      "| 19|oh dear were you ...|     0|    11|\n",
      "+---+--------------------+------+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tweets_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3749"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.filter(\"text is null\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_df = tweets_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1596251"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data in train, val and test sets\n",
    "(train_set, val_set, test_set) = tweets_df.randomSplit([0.98, 0.01, 0.01], seed=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data : 1564468 \n",
      "val data : 15789 \n",
      "test data : 15994\n"
     ]
    }
   ],
   "source": [
    "print(f\"train data : {train_set.count()} \\nval data : {val_set.count()} \\ntest data : {test_set.count()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(inputCol=\"text\", outputCol=\"words\")\n",
    "hashtf = HashingTF(numFeatures=2**16, inputCol=\"words\", outputCol=\"raw_features\")\n",
    "idf = IDF(inputCol=\"raw_features\", outputCol=\"features\", minDocFreq=5)\n",
    "label_stringIdx = StringIndexer(inputCol=\"target\", outputCol=\"label\")\n",
    "lr = LogisticRegression(maxIter=100)\n",
    "pipeline = Pipeline(stages=[tokenizer, hashtf, idf, label_stringIdx, lr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = pipeline.fit(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_pred = model.transform(val_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+--------------------+------+------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+----------+\n",
      "|   _c0|                text|target|length|               words|        raw_features|            features|label|       rawPrediction|         probability|prediction|\n",
      "+------+--------------------+------+------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+----------+\n",
      "|100097|another stinking ...|     0|     3|[another, stinkin...|(65536,[16307,245...|(65536,[16307,245...|  0.0|[3.02997935437078...|[0.95391026527641...|       0.0|\n",
      "|100117|finished work yay...|     0|     7|[finished, work, ...|(65536,[5877,8709...|(65536,[5877,8709...|  0.0|[0.89484455413591...|[0.70988890582443...|       0.0|\n",
      "|100129|why am i always i...|     0|     9|[why, am, i, alwa...|(65536,[9408,1424...|(65536,[9408,1424...|  0.0|[1.86205098113115...|[0.86553582753692...|       0.0|\n",
      "|100169|is getting over h...|     0|     8|[is, getting, ove...|(65536,[7209,2496...|(65536,[7209,2496...|  0.0|[1.15582270825097...|[0.76057285074887...|       0.0|\n",
      "|  1002|definitely no vac...|     0|     5|[definitely, no, ...|(65536,[6346,4124...|(65536,[6346,4124...|  0.0|[0.37086637130013...|[0.59166830768323...|       0.0|\n",
      "|100201|oh shitt i comple...|     0|    18|[oh, shitt, i, co...|(65536,[18184,190...|(65536,[18184,190...|  0.0|[1.57346884121818...|[0.82827755687761...|       0.0|\n",
      "|100269|sorri about that ...|     0|     9|[sorri, about, th...|(65536,[9958,1870...|(65536,[9958,1870...|  0.0|[2.79621946186693...|[0.94247118872326...|       0.0|\n",
      "|100296| i miss gabby cullen|     0|     4|[i, miss, gabby, ...|(65536,[5864,1903...|(65536,[5864,1903...|  0.0|[1.35852342762891...|[0.79551961121292...|       0.0|\n",
      "|100386|almost forgot i h...|     0|    17|[almost, forgot, ...|(65536,[9347,1899...|(65536,[9347,1899...|  0.0|[0.58843345872758...|[0.64300562771447...|       0.0|\n",
      "|100518|     my throat hurts|     0|     3| [my, throat, hurts]|(65536,[44336,518...|(65536,[44336,518...|  0.0|[3.00304322372066...|[0.95271142030682...|       0.0|\n",
      "|100558|i missed most of ...|     0|    17|[i, missed, most,...|(65536,[19036,191...|(65536,[19036,191...|  0.0|[2.02711762829537...|[0.88361498374109...|       0.0|\n",
      "|100662|is mad at life mo...|     0|    11|[is, mad, at, lif...|(65536,[3639,1216...|(65536,[3639,1216...|  0.0|[1.83335821137859...|[0.86216129954852...|       0.0|\n",
      "|100774|             not yet|     0|     2|          [not, yet]|(65536,[25085,273...|(65536,[25085,273...|  0.0|[0.64936797356343...|[0.65686802283020...|       0.0|\n",
      "|100808|was thinking abou...|     0|    14|[was, thinking, a...|(65536,[3639,6783...|(65536,[3639,6783...|  0.0|[1.43920171771370...|[0.80833100241392...|       0.0|\n",
      "|100839|could not make wi...|     0|    12|[could, not, make...|(65536,[18354,241...|(65536,[18354,241...|  0.0|[2.16649593944459...|[0.89720022964121...|       0.0|\n",
      "|100926|awaaaake fuck my ...|     0|    25|[awaaaake, fuck, ...|(65536,[6575,8317...|(65536,[6575,8317...|  0.0|[-0.9501209447285...|[0.27886049966569...|       1.0|\n",
      "|101018|watching some dra...|     0|     5|[watching, some, ...|(65536,[12338,192...|(65536,[12338,192...|  0.0|[0.80057222898702...|[0.69009687314812...|       0.0|\n",
      "|101248|the caps are out ...|     0|     8|[the, caps, are, ...|(65536,[1310,6094...|(65536,[1310,6094...|  0.0|[-0.6412055909135...|[0.34497406478874...|       1.0|\n",
      "|101380|i wish that was e...|     0|     8|[i, wish, that, w...|(65536,[19036,207...|(65536,[19036,207...|  0.0|[0.56126472117385...|[0.63674512182636...|       0.0|\n",
      "|101494|so little time so...|     0|     8|[so, little, time...|(65536,[11228,275...|(65536,[11228,275...|  0.0|[-0.2845660313530...|[0.42933470994649...|       1.0|\n",
      "+------+--------------------+------+------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "val_pred.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------------------------------------------------------------------------------------------------------+-----+----------+------------------------------------------+\n",
      "|text                                                                                                                               |label|prediction|rawPrediction                             |\n",
      "+-----------------------------------------------------------------------------------------------------------------------------------+-----+----------+------------------------------------------+\n",
      "|another stinking migraine                                                                                                          |0.0  |0.0       |[3.0299793543707803,-3.0299793543707803]  |\n",
      "|finished work yay time for assignments boo                                                                                         |0.0  |0.0       |[0.8948445541359162,-0.8948445541359162]  |\n",
      "|why am i always ill it drives me mad                                                                                               |0.0  |0.0       |[1.8620509811311563,-1.8620509811311563]  |\n",
      "|is getting over her am starts at work                                                                                              |0.0  |0.0       |[1.1558227082509724,-1.1558227082509724]  |\n",
      "|definitely no vacation for me                                                                                                      |0.0  |0.0       |[0.37086637130013833,-0.37086637130013833]|\n",
      "|oh shitt i completely forgot there was a sgtweetup today will have to meet u guys another time                                     |0.0  |0.0       |[1.5734688412181803,-1.5734688412181803]  |\n",
      "|sorri about that i am really really really bored                                                                                   |0.0  |0.0       |[2.7962194618669356,-2.7962194618669356]  |\n",
      "|i miss gabby cullen                                                                                                                |0.0  |0.0       |[1.3585234276289122,-1.3585234276289122]  |\n",
      "|almost forgot i have a doctor s appointment today workout will have to wait until this afternoon                                   |0.0  |0.0       |[0.5884334587275806,-0.5884334587275806]  |\n",
      "|my throat hurts                                                                                                                    |0.0  |0.0       |[3.003043223720667,-3.003043223720667]    |\n",
      "|i missed most of your interview sorting out something at this end and lost contact with sustnow                                    |0.0  |0.0       |[2.0271176282953736,-2.0271176282953736]  |\n",
      "|is mad at life money debts need to earn more hmph                                                                                  |0.0  |0.0       |[1.8333582113785933,-1.8333582113785933]  |\n",
      "|not yet                                                                                                                            |0.0  |0.0       |[0.6493679735634332,-0.6493679735634332]  |\n",
      "|was thinking about it but no way would i give my money to o                                                                        |0.0  |0.0       |[1.4392017177137015,-1.4392017177137015]  |\n",
      "|could not make wiredwedto unfortunately are you going to spinto tomorrow night                                                     |0.0  |0.0       |[2.1664959394445935,-2.1664959394445935]  |\n",
      "|awaaaake fuck my life i m soooo tired however chicago calls and i ma see dem boys in anarbor tonight after while exploring the city|0.0  |1.0       |[-0.9501209447285646,0.9501209447285646]  |\n",
      "|watching some drake josh sick                                                                                                      |0.0  |0.0       |[0.8005722289870216,-0.8005722289870216]  |\n",
      "|the caps are out it was a slaughter                                                                                                |0.0  |1.0       |[-0.6412055909135173,0.6412055909135173]  |\n",
      "|i wish that was easier done than said                                                                                              |0.0  |0.0       |[0.5612647211738593,-0.5612647211738593]  |\n",
      "|so little time so much to do xxxx                                                                                                  |0.0  |1.0       |[-0.2845660313530859,0.2845660313530859]  |\n",
      "+-----------------------------------------------------------------------------------------------------------------------------------+-----+----------+------------------------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "val_pred.select(\"text\", \"label\", \"prediction\", \"rawPrediction\").show(20, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7899366175535865"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator = BinaryClassificationEvaluator(rawPredictionCol='prediction')\n",
    "evaluator.evaluate(val_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7900437013110393"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = val_pred.filter(val_pred.label == val_pred.prediction).count() / float(val_set.count())\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('tweets_analyzer.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}