Sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Sentiment analysis.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNSRlr5zKAF4M1H7doQlOuP",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sensrdt/Sentiment-Analyzer/blob/master/Sentiment_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i3E5O834kY9i",
        "colab_type": "text"
      },
      "source": [
        "# NLTk install and Downloading the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBnFegOZ9Lzq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import nltk"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1aN4VRza-k5D",
        "colab_type": "code",
        "outputId": "439d176f-2385-4063-d7b8-732f9ca68a56",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "nltk.download('twitter_samples')\n",
        "nltk.download('punkt')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('stopwords')"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package twitter_samples to /root/nltk_data...\n",
            "[nltk_data]   Package twitter_samples is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rj3zrJEhknBW",
        "colab_type": "text"
      },
      "source": [
        "# Tokenizing the Data\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ur7B7pU4-4X7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.corpus import twitter_samples\n",
        "tp = twitter_samples.strings('positive_tweets.json')\n",
        "tn = twitter_samples.strings('negative_tweets.json')\n",
        "ns = twitter_samples.strings('tweets.20150430-223406.json')\n",
        "break_into_tokens = twitter_samples.tokenized('positive_tweets.json')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Iu1iz8P_WBT",
        "colab_type": "code",
        "outputId": "87843cf9-3201-4185-cddf-34dcf2640d97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from nltk.tag import pos_tag\n",
        "mapping = twitter_samples.tokenized('positive_tweets.json')\n",
        "print(pos_tag(mapping[0]))"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('#FollowFriday', 'JJ'), ('@France_Inte', 'NNP'), ('@PKuchly57', 'NNP'), ('@Milipol_Paris', 'NNP'), ('for', 'IN'), ('being', 'VBG'), ('top', 'JJ'), ('engaged', 'VBN'), ('members', 'NNS'), ('in', 'IN'), ('my', 'PRP$'), ('community', 'NN'), ('this', 'DT'), ('week', 'NN'), (':)', 'NN')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZJvFJVm5k9xL",
        "colab_type": "text"
      },
      "source": [
        "# Normalisation of the Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FbWPDMTbA1f-",
        "colab_type": "code",
        "outputId": "e567fafd-07cc-46d9-a576-3d384ee7f0a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "def lemmatizing_sentence(tt):\n",
        "  lemmatizer = WordNetLemmatizer()\n",
        "  lemmatized_sentence = []\n",
        "  for word, tag in pos_tag(tt):\n",
        "    if tag.startswith('NN'):\n",
        "      pos = 'n'\n",
        "    elif tag.startswith('VB'):\n",
        "      pos = 'v'\n",
        "    else:\n",
        "      pos = 'a'\n",
        "    lemmatized_sentence.append(lemmatizer.lemmatize(word, pos))\n",
        "  return lemmatized_sentence\n",
        "\n",
        "print(lemmatizing_sentence(mapping[0]))"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['#FollowFriday', '@France_Inte', '@PKuchly57', '@Milipol_Paris', 'for', 'be', 'top', 'engage', 'member', 'in', 'my', 'community', 'this', 'week', ':)']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zG70EKyglLA_",
        "colab_type": "text"
      },
      "source": [
        "# Removing/cleaning the noise \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yXlyHH-aCM2A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import string, re\n",
        "def remove_noise(mapping, stop_words = ()):\n",
        "\n",
        "    cleaned_tokens = []\n",
        "\n",
        "    for token, tag in pos_tag(mapping):\n",
        "        token = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\\(\\),]|'\\\n",
        "                       '(?:%[0-9a-fA-F][0-9a-fA-F]))+','', token)\n",
        "        token = re.sub(\"(@[A-Za-z0-9_]+)\",\"\", token)\n",
        "\n",
        "        if tag.startswith(\"NN\"):\n",
        "            pos = 'n'\n",
        "        elif tag.startswith('VB'):\n",
        "            pos = 'v'\n",
        "        else:\n",
        "            pos = 'a'\n",
        "\n",
        "        lemmatizer = WordNetLemmatizer()\n",
        "        token = lemmatizer.lemmatize(token, pos)\n",
        "\n",
        "        if len(token) > 0 and token not in string.punctuation and token.lower() not in stop_words:\n",
        "            cleaned_tokens.append(token.lower())\n",
        "    return cleaned_tokens"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNVsOx3JI2SK",
        "colab_type": "code",
        "outputId": "2ff808e5-d3ff-46a3-b426-203aa65cf06f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from nltk.corpus import stopwords\n",
        "print(stopwords.words('english'))"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bSO7jmINJQIv",
        "colab_type": "code",
        "outputId": "fd061b5d-87aa-4c93-b6b4-1ac51d41dc23",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(remove_noise(mapping[0], stopwords.words('english')))"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['#followfriday', 'top', 'engage', 'member', 'community', 'week', ':)']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMbZVt9-JZxE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tpt = twitter_samples.tokenized('positive_tweets.json')\n",
        "npt = twitter_samples.tokenized('negative_tweets.json')\n",
        "stpw = stopwords.words('english')\n",
        "\n",
        "pctl = []\n",
        "nctl = []\n",
        "\n",
        "for tokens in tpt:\n",
        "  pctl.append(remove_noise(tokens, stpw))\n",
        "for tokens in npt:\n",
        "  nctl.append(remove_noise(tokens, stpw))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z--O4Jn8ljBW",
        "colab_type": "text"
      },
      "source": [
        "# Frequency of words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o_OEoFokUqAG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_all_words(cleaned_tokens_list):\n",
        "    for tokens in cleaned_tokens_list:\n",
        "        for token in tokens:\n",
        "            yield token\n",
        "\n",
        "all_pos_words = get_all_words(pctl)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bUbtWQVLW7oR",
        "colab_type": "code",
        "outputId": "eb7df573-2ad7-4901-a341-88bbc1d682e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from nltk import FreqDist\n",
        "\n",
        "freq_dist_pos = FreqDist(all_pos_words)\n",
        "print(freq_dist_pos.most_common(10))"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(':)', 3691), (':-)', 701), (':d', 658), ('thanks', 388), ('follow', 357), ('love', 333), ('...', 290), ('good', 283), ('get', 263), ('thank', 253)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T336bpcymCv0",
        "colab_type": "text"
      },
      "source": [
        "# Preparing the data "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MPpJV2SyXeU3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_all_words(cleaned_tokens_list):\n",
        "    for tokens in cleaned_tokens_list:\n",
        "        for token in tokens:\n",
        "            yield token\n",
        "\n",
        "all_pos_words = get_all_words(pctl)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6I2P0_lAb8la",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_tweets_for_model(cleaned_tokens_list):\n",
        "    for tweet_tokens in cleaned_tokens_list:\n",
        "        yield dict([token, True] for token in tweet_tokens)\n",
        "\n",
        "positive_tokens_for_model = get_tweets_for_model(pctl)\n",
        "negative_tokens_for_model = get_tweets_for_model(nctl)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfAjcSY1mJkw",
        "colab_type": "text"
      },
      "source": [
        "## Spliting data for training and testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3n82w5VzcqZY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "\n",
        "positive_dataset = [(tweet_dict, \"Positive\")\n",
        "for tweet_dict in positive_tokens_for_model]\n",
        "\n",
        "negative_dataset = [(tweet_dict, \"Negative\")\n",
        "for tweet_dict in negative_tokens_for_model]\n",
        "\n",
        "dataset = positive_dataset + negative_dataset\n",
        "\n",
        "random.shuffle(dataset)\n",
        "\n",
        "train_data = dataset[:7000]\n",
        "test_data = dataset[7000:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B-E_k3Mrn7Tq",
        "colab_type": "text"
      },
      "source": [
        "# Train the data and then Test the data on a dummy data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pk5dyk8YfINb",
        "colab_type": "code",
        "outputId": "a167222d-6f80-4995-f085-cf6e88d55786",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "from nltk import classify\n",
        "from nltk import NaiveBayesClassifier\n",
        "classifier = NaiveBayesClassifier.train(train_data)\n",
        "\n",
        "print(\"Accuracy is :\", classify.accuracy(classifier, test_data))\n",
        "\n",
        "print(classifier.show_most_informative_features(10))"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy is : 0.997\n",
            "Most Informative Features\n",
            "                      :( = True           Negati : Positi =   2046.4 : 1.0\n",
            "                      :) = True           Positi : Negati =   1657.3 : 1.0\n",
            "                follower = True           Positi : Negati =     35.9 : 1.0\n",
            "                     x15 = True           Negati : Positi =     19.5 : 1.0\n",
            "                     bam = True           Positi : Negati =     19.1 : 1.0\n",
            "                 welcome = True           Positi : Negati =     17.5 : 1.0\n",
            "                     sad = True           Negati : Positi =     17.3 : 1.0\n",
            "              appreciate = True           Positi : Negati =     15.1 : 1.0\n",
            "                    blog = True           Positi : Negati =     15.1 : 1.0\n",
            "               community = True           Positi : Negati =     13.8 : 1.0\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aa_eBoH9gkQF",
        "colab_type": "code",
        "outputId": "ace8d0f8-09b5-40fd-ea67-3967c4468a6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "ctweet = \"Just In: 38 CISF personnel from Kolkata have tested positive. They were all quarantined as they were contacts of one CISF personnel who had died of Covid-19. Total 39 positive cases so far.\"\n",
        "ctweet1 = \"Many countries are taking drastic measures in a bid to halt the spread of the virus, including social distancing, closing bars, restaurants and schools or, in the case of places like Italy, putting the entire community on lockdown.\"\n",
        "ctweet2 = \"The Indian Army is considering a proposal to attract young working professionals to join the force for a three-year tenure as officers and in other ranks for a variety of roles. The Tour of Duty (ToD) for three years is being mulled in the wake of resurgence of nationalism and patriotism in the country. \"\n",
        "ctweet3 = \"Security forces in Burkina Faso killed 20 terrorists in a clash near the border with Niger, the army said Wednesday. \"\n",
        "ctoken = remove_noise(word_tokenize(ctweet3))\n",
        "print(classifier.classify(dict([token, True] for token in ctoken)))"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pZHy_SXOqxt2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}