sample_NLP_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = [\"this\", \"that\", \"than\", \"a\", \"the\", \"i\", \"im\", \"ive\", \"is\", \"was\", \"for\"]\n",
    "all_comments = [[\"This movie is the best.\", #Positive\n",
    "             \"The best movie I've ever seen!\",\n",
    "             \"betteR than the rest.\",\n",
    "             \"Better tHan the first version.\",\n",
    "             \"I'm just here for the comments. :)\"],\n",
    "            [\"I HATE THIS MOVIE. aaaarrrg!\", #Negative\n",
    "             \"What's this garbage!\",\n",
    "             \"This was worse than I thought\",\n",
    "             \"Worse movie ever produced.\",\n",
    "             \"A terrible movie!\",]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[['movie', 'best'], ['best', 'movie', 'ever', 'seen'], ['better', 'rest'], ['better', 'first', 'version'], ['just', 'here', 'comments']], [['hate', 'movie', 'aaaarrrg'], ['whats', 'garbage'], ['worse', 'thought'], ['worse', 'movie', 'ever', 'produced'], ['terrible', 'movie']]] \n\nNumber of Features: 27\n"
     ]
    }
   ],
   "source": [
    "clean_comments = []\n",
    "features = []\n",
    "\n",
    "for comments in all_comments:\n",
    "    temp_= []\n",
    "    for comment in comments:\n",
    "        cleaned_tokens = []\n",
    "        tokens = comment.split()\n",
    "        for item in tokens:\n",
    "            item = item.lower()\n",
    "            item = re.sub('[^a-zA-Z]', '', item)\n",
    "            if item not in stopwords and len(item) > 1:\n",
    "                cleaned_tokens.append(item)\n",
    "                features.append(item)\n",
    "        temp_.append(cleaned_tokens)\n",
    "    clean_comments.append(temp_)\n",
    "print(clean_comments, \"\\n\\nNumber of Features:\", len(features))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Est Probability of Each Feature/Token (in each class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['movie', 'best', 'best', 'movie', 'ever', 'seen', 'better', 'rest', 'better', 'first', 'version', 'just', 'here', 'comments']\n['hate', 'movie', 'aaaarrrg', 'whats', 'garbage', 'worse', 'thought', 'worse', 'movie', 'ever', 'produced', 'terrible', 'movie'] \n\n[0.14285714285714285, 0.14285714285714285, 0.14285714285714285, 0.14285714285714285, 0.07142857142857142, 0.07142857142857142, 0.14285714285714285, 0.07142857142857142, 0.14285714285714285, 0.07142857142857142, 0.07142857142857142, 0.07142857142857142, 0.07142857142857142, 0.07142857142857142, 0.024390243902439025, 0.14285714285714285, 0.024390243902439025, 0.024390243902439025, 0.024390243902439025, 0.024390243902439025, 0.024390243902439025, 0.024390243902439025, 0.14285714285714285, 0.07142857142857142, 0.024390243902439025, 0.024390243902439025, 0.14285714285714285] \n\n [0.23076923076923078, 0.025, 0.025, 0.23076923076923078, 0.07692307692307693, 0.025, 0.025, 0.025, 0.025, 0.025, 0.025, 0.025, 0.025, 0.025, 0.07692307692307693, 0.23076923076923078, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.15384615384615385, 0.07692307692307693, 0.15384615384615385, 0.23076923076923078, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.23076923076923078]\n"
     ]
    }
   ],
   "source": [
    "p_comments = []\n",
    "n_comments = []\n",
    "\n",
    "for docs in clean_comments[0]:\n",
    "    p_comments.extend(docs)\n",
    "print(p_comments)\n",
    "for docs in clean_comments[1]:\n",
    "    n_comments.extend(docs)\n",
    "print(n_comments, \"\\n\")\n",
    "\n",
    "p_features = []\n",
    "n_features = []\n",
    "\n",
    "#Positive class features\n",
    "for f in features:\n",
    "    count = p_comments.count(f)\n",
    "    if count != 0:\n",
    "        prob = count/(len(p_comments))\n",
    "    else:\n",
    "        prob = (count+1)/(len(p_comments) + len(features)*1)\n",
    "    p_features.append(prob)\n",
    "    #print(f, count, prob)\n",
    "\n",
    "#Negative class features\n",
    "for f in features:\n",
    "    count = n_comments.count(f)\n",
    "    if count != 0:\n",
    "        prob = count/(len(n_comments))\n",
    "    else:\n",
    "        prob = (count+1)/(len(n_comments) + len(features)*1)\n",
    "    n_features.append(prob)\n",
    "    #print(f, count, prob)\n",
    "    \n",
    "print(p_features, \"\\n\\n\", n_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting to Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Positive vectors: [[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1], [1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]] \n\nNegative vectors: [[1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0], [1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1], [1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1]]\n"
     ]
    }
   ],
   "source": [
    "p_vector = []\n",
    "n_vector = []\n",
    "\n",
    "#Positives\n",
    "for doc in clean_comments[0]:\n",
    "    temp_ = []\n",
    "    for token in features:\n",
    "        if token in doc:\n",
    "            temp_.append(1)\n",
    "        else:\n",
    "            temp_.append(0)\n",
    "    p_vector.append(temp_)\n",
    "\n",
    "#Negatives\n",
    "for doc in clean_comments[1]:\n",
    "    temp_ = []\n",
    "    for token in features:\n",
    "        if token in doc:\n",
    "            temp_.append(1)\n",
    "        else:\n",
    "            temp_.append(0)\n",
    "    n_vector.append(temp_)\n",
    "print(\"Positive vectors:\", p_vector, \"\\n\\nNegative vectors:\" ,n_vector)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Est Probability of Each Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Total documents: 10\n"
     ]
    }
   ],
   "source": [
    "n_total_docs = len(p_vector) + len(n_vector)\n",
    "print(\"Total documents:\", n_total_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Probability of each class: 0.5 0.5\n"
     ]
    }
   ],
   "source": [
    "p_probability = len(p_vector)/n_total_docs\n",
    "n_probability = len(n_vector)/n_total_docs\n",
    "print(\"Probability of each class:\", p_probability, n_probability)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict Training Data Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Worse movie ever produced.\nNegative Comment\n"
     ]
    }
   ],
   "source": [
    "training_vector = n_vector[3]\n",
    "print(all_comments[1][3])\n",
    "\n",
    "p_prob = 1\n",
    "n_prob = 1\n",
    "\n",
    "for i, item in enumerate(training_vector):\n",
    "    if item == 1:\n",
    "        p_prob = p_prob * p_features[i]\n",
    "        n_prob = n_prob * n_features[i]\n",
    "p_prob = p_prob * p_probability\n",
    "n_prob = n_prob * n_probability\n",
    "\n",
    "if p_prob > n_prob:\n",
    "    print(\"Positive Comment\")\n",
    "else:\n",
    "    print(\"Negative Comment\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict New Data Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Is not this movie the best of all?\nPositive Comment\n"
     ]
    }
   ],
   "source": [
    "test_comment = \"Is not this movie the best of all?\"\n",
    "print(test_comment)\n",
    "\n",
    "tokenized_comment = []\n",
    "tokens = test_comment.split()\n",
    "for item in tokens:\n",
    "    item = item.lower()\n",
    "    item = re.sub('[^a-zA-Z]', '', item)\n",
    "    if item not in stopwords and len(item) > 1:\n",
    "        tokenized_comment.append(item)\n",
    "\n",
    "test_vector = []\n",
    "for token in features:\n",
    "    if token in tokenized_comment:\n",
    "        test_vector.append(1)\n",
    "    else:\n",
    "        test_vector.append(0)\n",
    "\n",
    "test_p_class = 1\n",
    "test_n_class = 1\n",
    "\n",
    "for i, item in enumerate(test_vector):\n",
    "    if item == 1:\n",
    "        test_p_class = test_p_class * p_features[i]\n",
    "        test_n_class = test_n_class * n_features[i]\n",
    "        \n",
    "test_p_class = test_p_class * p_probability\n",
    "test_n_class = test_n_class * n_probability\n",
    "\n",
    "\n",
    "if test_p_class > test_n_class:\n",
    "    print(\"Positive Comment\")\n",
    "else:\n",
    "    print(\"Negative Comment\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "Python 3.7.7 64-bit",
   "display_name": "Python 3.7.7 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
    }
   }
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}