Twitter_sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis of Twitter Data using PySpark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName('Twitter_sentiment_analysis').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = spark.read.csv('twitter_dataset.csv', inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+----------+--------------------+--------+---------------+--------------------+\n",
      "|_c0|       _c1|                 _c2|     _c3|            _c4|                 _c5|\n",
      "+---+----------+--------------------+--------+---------------+--------------------+\n",
      "|  0|1467810369|Mon Apr 06 22:19:...|NO_QUERY|_TheSpecialOne_|@switchfoot http:...|\n",
      "|  0|1467810672|Mon Apr 06 22:19:...|NO_QUERY|  scotthamilton|is upset that he ...|\n",
      "|  0|1467810917|Mon Apr 06 22:19:...|NO_QUERY|       mattycus|@Kenichan I dived...|\n",
      "|  0|1467811184|Mon Apr 06 22:19:...|NO_QUERY|        ElleCTF|my whole body fee...|\n",
      "|  0|1467811193|Mon Apr 06 22:19:...|NO_QUERY|         Karoli|@nationwideclass ...|\n",
      "|  0|1467811372|Mon Apr 06 22:20:...|NO_QUERY|       joy_wolf|@Kwesidei not the...|\n",
      "|  0|1467811592|Mon Apr 06 22:20:...|NO_QUERY|        mybirch|         Need a hug |\n",
      "|  0|1467811594|Mon Apr 06 22:20:...|NO_QUERY|           coZZ|@LOLTrish hey  lo...|\n",
      "|  0|1467811795|Mon Apr 06 22:20:...|NO_QUERY|2Hood4Hollywood|@Tatiana_K nope t...|\n",
      "|  0|1467812025|Mon Apr 06 22:20:...|NO_QUERY|        mimismo|@twittera que me ...|\n",
      "|  0|1467812416|Mon Apr 06 22:20:...|NO_QUERY| erinx3leannexo|spring break in p...|\n",
      "|  0|1467812579|Mon Apr 06 22:20:...|NO_QUERY|   pardonlauren|I just re-pierced...|\n",
      "|  0|1467812723|Mon Apr 06 22:20:...|NO_QUERY|           TLeC|@caregiving I cou...|\n",
      "|  0|1467812771|Mon Apr 06 22:20:...|NO_QUERY|robrobbierobert|@octolinz16 It it...|\n",
      "|  0|1467812784|Mon Apr 06 22:20:...|NO_QUERY|    bayofwolves|@smarrison i woul...|\n",
      "|  0|1467812799|Mon Apr 06 22:20:...|NO_QUERY|     HairByJess|@iamjazzyfizzle I...|\n",
      "|  0|1467812964|Mon Apr 06 22:20:...|NO_QUERY| lovesongwriter|Hollis' death sce...|\n",
      "|  0|1467813137|Mon Apr 06 22:20:...|NO_QUERY|       armotley|about to file taxes |\n",
      "|  0|1467813579|Mon Apr 06 22:20:...|NO_QUERY|     starkissed|@LettyA ahh ive a...|\n",
      "|  0|1467813782|Mon Apr 06 22:20:...|NO_QUERY|      gi_gi_bee|@FakerPattyPattz ...|\n",
      "+---+----------+--------------------+--------+---------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- _c0: integer (nullable = true)\n",
      " |-- _c1: long (nullable = true)\n",
      " |-- _c2: string (nullable = true)\n",
      " |-- _c3: string (nullable = true)\n",
      " |-- _c4: string (nullable = true)\n",
      " |-- _c5: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import col\n",
    "from pyspark.sql.types import StringType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.withColumnRenamed('_c0','target').withColumnRenamed('_c5','text')\n",
    "data = data.withColumn('target', col('target').cast(StringType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+----------+--------------------+--------+---------------+--------------------+\n",
      "|target|       _c1|                 _c2|     _c3|            _c4|                text|\n",
      "+------+----------+--------------------+--------+---------------+--------------------+\n",
      "|     0|1467810369|Mon Apr 06 22:19:...|NO_QUERY|_TheSpecialOne_|@switchfoot http:...|\n",
      "|     0|1467810672|Mon Apr 06 22:19:...|NO_QUERY|  scotthamilton|is upset that he ...|\n",
      "|     0|1467810917|Mon Apr 06 22:19:...|NO_QUERY|       mattycus|@Kenichan I dived...|\n",
      "|     0|1467811184|Mon Apr 06 22:19:...|NO_QUERY|        ElleCTF|my whole body fee...|\n",
      "|     0|1467811193|Mon Apr 06 22:19:...|NO_QUERY|         Karoli|@nationwideclass ...|\n",
      "|     0|1467811372|Mon Apr 06 22:20:...|NO_QUERY|       joy_wolf|@Kwesidei not the...|\n",
      "|     0|1467811592|Mon Apr 06 22:20:...|NO_QUERY|        mybirch|         Need a hug |\n",
      "|     0|1467811594|Mon Apr 06 22:20:...|NO_QUERY|           coZZ|@LOLTrish hey  lo...|\n",
      "|     0|1467811795|Mon Apr 06 22:20:...|NO_QUERY|2Hood4Hollywood|@Tatiana_K nope t...|\n",
      "|     0|1467812025|Mon Apr 06 22:20:...|NO_QUERY|        mimismo|@twittera que me ...|\n",
      "|     0|1467812416|Mon Apr 06 22:20:...|NO_QUERY| erinx3leannexo|spring break in p...|\n",
      "|     0|1467812579|Mon Apr 06 22:20:...|NO_QUERY|   pardonlauren|I just re-pierced...|\n",
      "|     0|1467812723|Mon Apr 06 22:20:...|NO_QUERY|           TLeC|@caregiving I cou...|\n",
      "|     0|1467812771|Mon Apr 06 22:20:...|NO_QUERY|robrobbierobert|@octolinz16 It it...|\n",
      "|     0|1467812784|Mon Apr 06 22:20:...|NO_QUERY|    bayofwolves|@smarrison i woul...|\n",
      "|     0|1467812799|Mon Apr 06 22:20:...|NO_QUERY|     HairByJess|@iamjazzyfizzle I...|\n",
      "|     0|1467812964|Mon Apr 06 22:20:...|NO_QUERY| lovesongwriter|Hollis' death sce...|\n",
      "|     0|1467813137|Mon Apr 06 22:20:...|NO_QUERY|       armotley|about to file taxes |\n",
      "|     0|1467813579|Mon Apr 06 22:20:...|NO_QUERY|     starkissed|@LettyA ahh ive a...|\n",
      "|     0|1467813782|Mon Apr 06 22:20:...|NO_QUERY|      gi_gi_bee|@FakerPattyPattz ...|\n",
      "+------+----------+--------------------+--------+---------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- target: string (nullable = true)\n",
      " |-- _c1: long (nullable = true)\n",
      " |-- _c2: string (nullable = true)\n",
      " |-- _c3: string (nullable = true)\n",
      " |-- _c4: string (nullable = true)\n",
      " |-- text: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Selecting only few records from the total data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import rand\n",
    "\n",
    "data = data.orderBy(rand()).limit(600000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.select('target','text')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing special characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import col, lower, regexp_replace, split\n",
    "\n",
    "def clean_text(c):\n",
    "    c = lower(c)\n",
    "    c = regexp_replace(c, \"^rt \", \"\")\n",
    "    c = regexp_replace(c, \"(https?\\://)\\S+\", \"\")\n",
    "    c = regexp_replace(c, \"[^a-zA-Z0-9\\\\s]\", \"\")\n",
    "    #c = split(c, \"\\\\s+\") tokenization...\n",
    "    return c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_df = data.select(clean_text(col(\"text\")).alias(\"text\"),'target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+\n",
      "|                text|target|\n",
      "+--------------------+------+\n",
      "|having coffee and...|     4|\n",
      "|strawberry banana...|     4|\n",
      "|enjoying amp conf...|     4|\n",
      "|loljohnjk oh okay...|     4|\n",
      "|kwymore09 yay  to...|     4|\n",
      "|geminitwisted im ...|     4|\n",
      "|running late for ...|     0|\n",
      "|thanks for my new...|     4|\n",
      "|insidebooks ah i ...|     4|\n",
      "|   mtv movie awards |     4|\n",
      "|going to tenerife...|     0|\n",
      "|ladytwiglet thank...|     4|\n",
      "|how am i meant to...|     0|\n",
      "|im sorry j3ssucka...|     0|\n",
      "|done swimming not...|     0|\n",
      "|mrssosbourne welc...|     4|\n",
      "|the hangover is p...|     4|\n",
      "|amanilouella bad ...|     0|\n",
      "|fbmook  aww thnx ...|     4|\n",
      "|great idea tennis...|     4|\n",
      "+--------------------+------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clean_df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import Tokenizer,StopWordsRemover, CountVectorizer,IDF,StringIndexer\n",
    "\n",
    "tokenizer = Tokenizer(inputCol=\"text\", outputCol=\"token_text\")\n",
    "stopremove = StopWordsRemover(inputCol='token_text',outputCol='stop_tokens')\n",
    "count_vec = CountVectorizer(inputCol='stop_tokens',outputCol='c_vec')\n",
    "idf = IDF(inputCol=\"c_vec\", outputCol=\"tf_idf\")\n",
    "label_stringIdx = StringIndexer(inputCol = \"target\", outputCol = \"label\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a pipeline to tokenize and extract features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "\n",
    "pipeline = Pipeline(stages=[tokenizer,stopremove,count_vec,idf,label_stringIdx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = pipeline.fit(clean_df).transform(clean_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = d.withColumnRenamed('tf_idf','features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = d.select('features','label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|            features|label|\n",
      "+--------------------+-----+\n",
      "|(262144,[12,117,2...|  1.0|\n",
      "|(262144,[1,49,53,...|  1.0|\n",
      "|(262144,[0,28,80,...|  1.0|\n",
      "|(262144,[1,5,16,2...|  1.0|\n",
      "|(262144,[0,13,42,...|  1.0|\n",
      "|(262144,[0,1,88,2...|  1.0|\n",
      "|(262144,[0,205,33...|  0.0|\n",
      "|(262144,[25,30,69...|  1.0|\n",
      "|(262144,[107,218,...|  1.0|\n",
      "|(262144,[131,604,...|  1.0|\n",
      "|(262144,[0,3,9,84...|  0.0|\n",
      "|(262144,[0,30,509...|  1.0|\n",
      "|(262144,[0,2,24,2...|  0.0|\n",
      "|(262144,[1,54,150...|  0.0|\n",
      "|(262144,[62,101,1...|  0.0|\n",
      "|(262144,[0,4,42,2...|  1.0|\n",
      "|(262144,[0,25,259...|  1.0|\n",
      "|(262144,[52,220,2...|  0.0|\n",
      "|(262144,[0,16,224...|  1.0|\n",
      "|(262144,[38,73,75...|  1.0|\n",
      "+--------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "d.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train,test = d.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on a Logistic Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrmodel = lr.fit(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_result = lrmodel.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-----+--------------------+--------------------+----------+\n",
      "|      features|label|       rawPrediction|         probability|prediction|\n",
      "+--------------+-----+--------------------+--------------------+----------+\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.1504192736087...|[0.46246592529082...|       1.0|\n",
      "+--------------+-----+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lr_result.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of model is: 0.7178642057053932\n"
     ]
    }
   ],
   "source": [
    "acc_eval = MulticlassClassificationEvaluator()\n",
    "acc = acc_eval.evaluate(lr_result)\n",
    "print(\"Accuracy of model is: {}\".format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluation on a Naive Bayes Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import NaiveBayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "nb = NaiveBayes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "nbmodel = nb.fit(train)\n",
    "\n",
    "nb_result = nbmodel.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-----+--------------------+--------------------+----------+\n",
      "|      features|label|       rawPrediction|         probability|prediction|\n",
      "+--------------+-----+--------------------+--------------------+----------+\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "|(262144,[],[])|  0.0|[-0.6937760125253...|[0.49968568285399...|       1.0|\n",
      "+--------------+-----+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "nb_result.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of model is: 0.7275316721987981\n"
     ]
    }
   ],
   "source": [
    "acc_eval = MulticlassClassificationEvaluator()\n",
    "acc = acc_eval.evaluate(nb_result)\n",
    "print(\"Accuracy of model is: {}\".format(acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}