sentiment_review.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "sentiment_review",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1l3oRJIUC49ciNhau2HZOOV5ICV_wr-KN",
      "authorship_tag": "ABX9TyPsxdKgmxy3WXmOMxlfkqwn",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rainmaker29/Sentiment-Classification/blob/master/sentiment_review.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0KHi7h7wSLzs",
        "colab_type": "text"
      },
      "source": [
        "# Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_m2zrM3JQZRJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# read data from text files\n",
        "with open('/content/drive/My Drive/sentiment_review/reviews.txt', 'r') as f:\n",
        "    reviews = f.read()\n",
        "with open('/content/drive/My Drive/sentiment_review/labels.txt', 'r') as f:\n",
        "    labels = f.read()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCve4mBCRFOt",
        "colab_type": "code",
        "outputId": "e83e72ed-cec4-43a1-bc16-42536e4fc09d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "print(reviews[:1000])\n",
        "print()\n",
        "print(labels[:20])"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "bromwell high is a cartoon comedy . it ran at the same time as some other programs about school life  such as  teachers  . my   years in the teaching profession lead me to believe that bromwell high  s satire is much closer to reality than is  teachers  . the scramble to survive financially  the insightful students who can see right through their pathetic teachers  pomp  the pettiness of the whole situation  all remind me of the schools i knew and their students . when i saw the episode in which a student repeatedly tried to burn down the school  i immediately recalled . . . . . . . . . at . . . . . . . . . . high . a classic line inspector i  m here to sack one of your teachers . student welcome to bromwell high . i expect that many adults of my age think that bromwell high is far fetched . what a pity that it isn  t   \n",
            "story of a man who has unnatural feelings for a pig . starts out with a opening scene that is a terrific example of absurd comedy . a formal orchestra audience is turn\n",
            "\n",
            "positive\n",
            "negative\n",
            "po\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zm11HM_bSPSF",
        "colab_type": "text"
      },
      "source": [
        "# Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7x4DIIB2RJOm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from string import punctuation\n",
        "\n",
        "# get rid of punctuation\n",
        "reviews = reviews.lower() # lowercase, standardize\n",
        "all_text = ''.join([c for c in reviews if c not in punctuation])\n",
        "\n",
        "# split by new lines and spaces\n",
        "reviews_split = all_text.split('\\n')\n",
        "all_text = ' '.join(reviews_split)\n",
        "\n",
        "# create a list of words\n",
        "words = all_text.split()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IQWyMwDpRVmX",
        "colab_type": "code",
        "outputId": "a9c3f48d-77b8-47bc-9bd4-f7f246292f11",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        }
      },
      "source": [
        "words[:30]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['bromwell',\n",
              " 'high',\n",
              " 'is',\n",
              " 'a',\n",
              " 'cartoon',\n",
              " 'comedy',\n",
              " 'it',\n",
              " 'ran',\n",
              " 'at',\n",
              " 'the',\n",
              " 'same',\n",
              " 'time',\n",
              " 'as',\n",
              " 'some',\n",
              " 'other',\n",
              " 'programs',\n",
              " 'about',\n",
              " 'school',\n",
              " 'life',\n",
              " 'such',\n",
              " 'as',\n",
              " 'teachers',\n",
              " 'my',\n",
              " 'years',\n",
              " 'in',\n",
              " 'the',\n",
              " 'teaching',\n",
              " 'profession',\n",
              " 'lead',\n",
              " 'me']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3GEVw1bRSX4V",
        "colab_type": "text"
      },
      "source": [
        "# Encoding words into integers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89ilkJi7RYNn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import Counter\n",
        "\n",
        "# A dictionary that maps words to integers\n",
        "counts = Counter(words)\n",
        "vocab = sorted(counts, key=counts.get, reverse=True)\n",
        "vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n",
        "\n",
        "# storing the tokenized reviews in reviews_ints\n",
        "reviews_ints = []\n",
        "for review in reviews_split:\n",
        "  reviews_ints.append([vocab_to_int[word] for word in review.split()])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7tgk_oQLRrHz",
        "colab_type": "code",
        "outputId": "a399c172-e494-485f-b417-a623ae0cd5cc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        }
      },
      "source": [
        "print('Unique words: ', len((vocab_to_int)))\n",
        "print()\n",
        "\n",
        "print('Tokenized review: \\n', reviews_ints[:1])"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Unique words:  74072\n",
            "\n",
            "Tokenized review: \n",
            " [[21025, 308, 6, 3, 1050, 207, 8, 2138, 32, 1, 171, 57, 15, 49, 81, 5785, 44, 382, 110, 140, 15, 5194, 60, 154, 9, 1, 4975, 5852, 475, 71, 5, 260, 12, 21025, 308, 13, 1978, 6, 74, 2395, 5, 613, 73, 6, 5194, 1, 24103, 5, 1983, 10166, 1, 5786, 1499, 36, 51, 66, 204, 145, 67, 1199, 5194, 19869, 1, 37442, 4, 1, 221, 883, 31, 2988, 71, 4, 1, 5787, 10, 686, 2, 67, 1499, 54, 10, 216, 1, 383, 9, 62, 3, 1406, 3686, 783, 5, 3483, 180, 1, 382, 10, 1212, 13583, 32, 308, 3, 349, 341, 2913, 10, 143, 127, 5, 7690, 30, 4, 129, 5194, 1406, 2326, 5, 21025, 308, 10, 528, 12, 109, 1448, 4, 60, 543, 102, 12, 21025, 308, 6, 227, 4146, 48, 3, 2211, 12, 8, 215, 23]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LI4uCWHCSckP",
        "colab_type": "text"
      },
      "source": [
        "# Encoding labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5klM-lqORz3n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 1=positive, 0=negative label conversion\n",
        "labels_split = labels.split('\\n')\n",
        "encoded_labels = np.array([1 if label == 'positive' else 0 for label in labels_split])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zKOTMcRhrk23",
        "colab_type": "text"
      },
      "source": [
        "# Outliers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jL-VRZpyrPRh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "1aaae8be-9602-4c7c-9118-538bba291ac2"
      },
      "source": [
        "# outlier review stats\n",
        "review_lens = Counter([len(x) for x in reviews_ints])\n",
        "print(\"Zero-length reviews: {}\".format(review_lens[0]))\n",
        "print(\"Maximum review length: {}\".format(max(review_lens)))\n",
        "\n",
        "print('Number of reviews before removing outliers: ', len(reviews_ints))\n",
        "\n",
        "## remove any reviews/labels with zero length from the reviews_ints list.\n",
        "\n",
        "# get indices of any reviews with length 0\n",
        "non_zero_idx = [ii for ii, review in enumerate(reviews_ints) if len(review) != 0]\n",
        "\n",
        "# remove 0-length reviews and their labels\n",
        "reviews_ints = [reviews_ints[ii] for ii in non_zero_idx]\n",
        "encoded_labels = np.array([encoded_labels[ii] for ii in non_zero_idx])\n",
        "\n",
        "print('Number of reviews after removing outliers: ', len(reviews_ints))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Zero-length reviews: 1\n",
            "Maximum review length: 2514\n",
            "Number of reviews before removing outliers:  25001\n",
            "Number of reviews after removing outliers:  25000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9vFwIdJyS0I_",
        "colab_type": "text"
      },
      "source": [
        "# Padding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "677UTzaJSGe2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def pad_features(reviews_ints, seq_length):\n",
        "  # This function creates a 2D matrix that stores each reviews as a row\n",
        "  # Since all the reviews have to be of same size,the cols are fixed and we \n",
        "  #pad the empty values\n",
        "\n",
        "  # getting the correct rows x cols shape\n",
        "  features = np.zeros((len(reviews_ints), seq_length), dtype=int)\n",
        "  # for each review, I grab that review and \n",
        "  for i, row in enumerate(reviews_ints):\n",
        "    features[i, -len(row):] = np.array(row)[:seq_length]\n",
        "  \n",
        "  return features"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4w-hYAbpbhe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        },
        "outputId": "7ef1eee3-44de-4e15-af9c-b6c7c153ebd2"
      },
      "source": [
        "seq_length = 200\n",
        "\n",
        "features = pad_features(reviews_ints, seq_length=seq_length)\n",
        "\n",
        "# print first 10 values of the first 30 batches \n",
        "print(features[:30,:10])"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [22382    42 46418    15   706 17139  3389    47    77    35]\n",
            " [ 4505   505    15     3  3342   162  8312  1652     6  4819]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   54    10    14   116    60   798   552    71   364     5]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    1   330   578    34     3   162   748  2731     9   325]\n",
            " [    9    11 10171  5305  1946   689   444    22   280   673]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    1   307 10399  2069  1565  6202  6528  3288 17946 10628]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   21   122  2069  1565   515  8181    88     6  1325  1182]\n",
            " [    1    20     6    76    40     6    58    81    95     5]\n",
            " [   54    10    84   329 26230 46427    63    10    14   614]\n",
            " [   11    20     6    30  1436 32317  3769   690 15100     6]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   40    26   109 17952  1422     9     1   327     4   125]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [   10   499     1   307 10399    55    74     8    13    30]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]\n",
            " [    0     0     0     0     0     0     0     0     0     0]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Hzv01nJsP88",
        "colab_type": "text"
      },
      "source": [
        "# Train-Validation-Test split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qa0WC9xCpk6j",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "7a14bdeb-3787-494f-d4f8-542cb67f84e6"
      },
      "source": [
        "split_frac = 0.8\n",
        "\n",
        "## split data into training, validation, and test data (features and labels, x and y)\n",
        "\n",
        "split_idx = int(len(features)*split_frac)\n",
        "train_x, remaining_x = features[:split_idx], features[split_idx:]\n",
        "train_y, remaining_y = encoded_labels[:split_idx], encoded_labels[split_idx:]\n",
        "\n",
        "test_idx = int(len(remaining_x)*0.5)\n",
        "val_x, test_x = remaining_x[:test_idx], remaining_x[test_idx:]\n",
        "val_y, test_y = remaining_y[:test_idx], remaining_y[test_idx:]\n",
        "\n",
        "## print out the shapes of your resultant feature data\n",
        "print(\"\\t\\t\\tFeature Shapes:\")\n",
        "print(\"Train set: \\t\\t{}\".format(train_x.shape), \n",
        "      \"\\nValidation set: \\t{}\".format(val_x.shape),\n",
        "      \"\\nTest set: \\t\\t{}\".format(test_x.shape))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\t\t\tFeature Shapes:\n",
            "Train set: \t\t(20000, 200) \n",
            "Validation set: \t(2500, 200) \n",
            "Test set: \t\t(2500, 200)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KduSz7Gisfs-",
        "colab_type": "text"
      },
      "source": [
        "# Creating datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jGeTDbLYsU4s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader\n",
        "\n",
        "# create Tensor datasets\n",
        "train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))\n",
        "valid_data = TensorDataset(torch.from_numpy(val_x), torch.from_numpy(val_y))\n",
        "test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))\n",
        "\n",
        "# dataloaders\n",
        "batch_size = 50\n",
        "\n",
        "# make sure the SHUFFLE your training data\n",
        "train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)\n",
        "valid_loader = DataLoader(valid_data, shuffle=True, batch_size=batch_size)\n",
        "test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pq8XLlURsjiV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        },
        "outputId": "d55c151a-4556-4614-8dee-c4b1f3995a50"
      },
      "source": [
        "# obtain one batch of training data\n",
        "dataiter = iter(train_loader)\n",
        "sample_x, sample_y = dataiter.next()\n",
        "\n",
        "print('Sample input size: ', sample_x.size()) # batch_size, seq_length\n",
        "print('Sample input: \\n', sample_x)\n",
        "print()\n",
        "print('Sample label size: ', sample_y.size()) # batch_size\n",
        "print('Sample label: \\n', sample_y)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample input size:  torch.Size([50, 200])\n",
            "Sample input: \n",
            " tensor([[    0,     0,     0,  ...,  4093,    90,   601],\n",
            "        [    0,  2538,  3353,  ...,   826,  1041, 52675],\n",
            "        [    0,     0,     0,  ..., 47504,   160,  6764],\n",
            "        ...,\n",
            "        [    1,  3356,     4,  ...,    19,    24,  2843],\n",
            "        [    0,     0,     0,  ...,   246,    73,  1553],\n",
            "        [  845,    69,   139,  ...,    32,     3,  3428]])\n",
            "\n",
            "Sample label size:  torch.Size([50])\n",
            "Sample label: \n",
            " tensor([0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0,\n",
            "        1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1,\n",
            "        0, 1])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kkdmtIWaspDG",
        "colab_type": "text"
      },
      "source": [
        "# Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XvKC5PD0skEI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e6b2a8fa-d361-45ee-b256-31fb3b56f6f4"
      },
      "source": [
        "# First checking if GPU is available\n",
        "train_on_gpu=torch.cuda.is_available()\n",
        "\n",
        "if(train_on_gpu):\n",
        "  print('Training on GPU.')\n",
        "else:\n",
        "  print('No GPU available, training on CPU.')"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training on GPU.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLfAp9STssUf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "\n",
        "class SentimentRNN(nn.Module):\n",
        "  def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):\n",
        "    \"\"\"\n",
        "    Initialize the model by setting up the layers.\n",
        "    \"\"\"\n",
        "    super(SentimentRNN, self).__init__()\n",
        "    self.output_size = output_size\n",
        "    self.n_layers = n_layers\n",
        "    self.hidden_dim = hidden_dim\n",
        "    \n",
        "    # embedding and LSTM layers\n",
        "    self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
        "    self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, \n",
        "                        dropout=drop_prob, batch_first=True)\n",
        "    \n",
        "    # dropout layer\n",
        "    self.dropout = nn.Dropout(0.3)\n",
        "    \n",
        "        # linear and sigmoid layers\n",
        "    self.fc = nn.Linear(hidden_dim, output_size)\n",
        "    self.sig = nn.Sigmoid()\n",
        "      \n",
        "  def forward(self, x, hidden):\n",
        "    \"\"\"\n",
        "    Perform a forward pass of our model on some input and hidden state.\n",
        "    \"\"\"\n",
        "    batch_size = x.size(0)\n",
        "    # embeddings and lstm_out\n",
        "    x = x.long()\n",
        "    embeds = self.embedding(x)\n",
        "    lstm_out, hidden = self.lstm(embeds, hidden)\n",
        "    # stack up lstm outputs\n",
        "    lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
        "        \n",
        "    # dropout and fully-connected layer\n",
        "    out = self.dropout(lstm_out)\n",
        "    out = self.fc(out)\n",
        "    # sigmoid function\n",
        "    sig_out = self.sig(out)\n",
        "        \n",
        "    # reshape to be batch_size first\n",
        "    sig_out = sig_out.view(batch_size, -1)\n",
        "    sig_out = sig_out[:, -1] # get last batch of labels\n",
        "        \n",
        "    # return last sigmoid output and hidden state\n",
        "    return sig_out, hidden\n",
        "    \n",
        "    \n",
        "  def init_hidden(self, batch_size):\n",
        "    '''' Initializes hidden state '''\n",
        "    # Create two new tensors with sizes n_layers x batch_size x hidden_dim,\n",
        "    # initialized to zero, for hidden state and cell state of LSTM\n",
        "    weight = next(self.parameters()).data\n",
        "    if (train_on_gpu):\n",
        "      hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),\n",
        "                  weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())\n",
        "    else:\n",
        "      hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),\n",
        "                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())\n",
        "        \n",
        "    return hidden\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "79Qn23ltu-v4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "d5c98e69-51df-4106-8e9c-e47089eb1112"
      },
      "source": [
        "vocab_size = len(vocab_to_int)+1 # +1 for the 0 padding + our word tokens\n",
        "output_size = 1\n",
        "embedding_dim = 400\n",
        "hidden_dim = 256\n",
        "n_layers = 2\n",
        "\n",
        "net = SentimentRNN(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)\n",
        "\n",
        "print(net)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SentimentRNN(\n",
            "  (embedding): Embedding(74073, 400)\n",
            "  (lstm): LSTM(400, 256, num_layers=2, batch_first=True, dropout=0.5)\n",
            "  (dropout): Dropout(p=0.3, inplace=False)\n",
            "  (fc): Linear(in_features=256, out_features=1, bias=True)\n",
            "  (sig): Sigmoid()\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N65I3vpCwKtF",
        "colab_type": "text"
      },
      "source": [
        "# Training\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2f30hFVOwH9o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lr=0.001\n",
        "\n",
        "criterion = nn.BCELoss()\n",
        "optimizer = torch.optim.Adam(net.parameters(), lr=lr)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4fZVu5PkwNkr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "159d34de-f9de-46a0-b9e4-d71ccc820fae"
      },
      "source": [
        "\n",
        "epochs = 4 # 3-4 is approx where I noticed the validation loss stop decreasing\n",
        "\n",
        "counter = 0\n",
        "print_every = 100\n",
        "clip=5 # gradient clipping\n",
        "\n",
        "# move model to GPU, if available\n",
        "if(train_on_gpu):\n",
        "    net.cuda()\n",
        "\n",
        "net.train()\n",
        "# train for some number of epochs\n",
        "for e in range(epochs):\n",
        "    # initialize hidden state\n",
        "    h = net.init_hidden(batch_size)\n",
        "\n",
        "    # batch loop\n",
        "    for inputs, labels in train_loader:\n",
        "        counter += 1\n",
        "\n",
        "        if(train_on_gpu):\n",
        "            inputs, labels = inputs.cuda(), labels.cuda()\n",
        "\n",
        "        # Creating new variables for the hidden state, otherwise\n",
        "        # we'd backprop through the entire training history\n",
        "        h = tuple([each.data for each in h])\n",
        "\n",
        "        # zero accumulated gradients\n",
        "        net.zero_grad()\n",
        "\n",
        "        # get the output from the model\n",
        "        output, h = net(inputs, h)\n",
        "\n",
        "        # calculate the loss and perform backprop\n",
        "        loss = criterion(output.squeeze(), labels.float())\n",
        "        loss.backward()\n",
        "        # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.\n",
        "        nn.utils.clip_grad_norm_(net.parameters(), clip)\n",
        "        optimizer.step()\n",
        "\n",
        "        # loss stats\n",
        "        if counter % print_every == 0:\n",
        "            # Get validation loss\n",
        "            val_h = net.init_hidden(batch_size)\n",
        "            val_losses = []\n",
        "            net.eval()\n",
        "            for inputs, labels in valid_loader:\n",
        "\n",
        "                # Creating new variables for the hidden state, otherwise\n",
        "                # we'd backprop through the entire training history\n",
        "                val_h = tuple([each.data for each in val_h])\n",
        "\n",
        "                if(train_on_gpu):\n",
        "                    inputs, labels = inputs.cuda(), labels.cuda()\n",
        "\n",
        "                output, val_h = net(inputs, val_h)\n",
        "                val_loss = criterion(output.squeeze(), labels.float())\n",
        "\n",
        "                val_losses.append(val_loss.item())\n",
        "\n",
        "            net.train()\n",
        "            print(\"Epoch: {}/{}...\".format(e+1, epochs),\n",
        "                  \"Step: {}...\".format(counter),\n",
        "                  \"Loss: {:.6f}...\".format(loss.item()),\n",
        "                  \"Val Loss: {:.6f}\".format(np.mean(val_losses)))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1/4... Step: 100... Loss: 0.705695... Val Loss: 0.659578\n",
            "Epoch: 1/4... Step: 200... Loss: 0.674316... Val Loss: 0.663448\n",
            "Epoch: 1/4... Step: 300... Loss: 0.665580... Val Loss: 0.609024\n",
            "Epoch: 1/4... Step: 400... Loss: 0.576278... Val Loss: 0.545834\n",
            "Epoch: 2/4... Step: 500... Loss: 0.545621... Val Loss: 0.533339\n",
            "Epoch: 2/4... Step: 600... Loss: 0.381714... Val Loss: 0.497611\n",
            "Epoch: 2/4... Step: 700... Loss: 0.415965... Val Loss: 0.485707\n",
            "Epoch: 2/4... Step: 800... Loss: 0.419560... Val Loss: 0.473547\n",
            "Epoch: 3/4... Step: 900... Loss: 0.276287... Val Loss: 0.509633\n",
            "Epoch: 3/4... Step: 1000... Loss: 0.252873... Val Loss: 0.496785\n",
            "Epoch: 3/4... Step: 1100... Loss: 0.266603... Val Loss: 0.463893\n",
            "Epoch: 3/4... Step: 1200... Loss: 0.364510... Val Loss: 0.458381\n",
            "Epoch: 4/4... Step: 1300... Loss: 0.096607... Val Loss: 0.488555\n",
            "Epoch: 4/4... Step: 1400... Loss: 0.170968... Val Loss: 0.509822\n",
            "Epoch: 4/4... Step: 1500... Loss: 0.246380... Val Loss: 0.480666\n",
            "Epoch: 4/4... Step: 1600... Loss: 0.185945... Val Loss: 0.503820\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gvILUww-yV2D",
        "colab_type": "text"
      },
      "source": [
        "# Testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "spLhSfM-w4OB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "59dab134-85c4-4071-a3a1-e6b3a9cd1185"
      },
      "source": [
        "\n",
        "test_losses = [] # track loss\n",
        "num_correct = 0\n",
        "\n",
        "# init hidden state\n",
        "h = net.init_hidden(batch_size)\n",
        "\n",
        "net.eval()\n",
        "# iterate over test data\n",
        "for inputs, labels in test_loader:\n",
        "\n",
        "    # Creating new variables for the hidden state, otherwise\n",
        "    # we'd backprop through the entire training history\n",
        "    h = tuple([each.data for each in h])\n",
        "\n",
        "    if(train_on_gpu):\n",
        "        inputs, labels = inputs.cuda(), labels.cuda()\n",
        "    \n",
        "    # get predicted outputs\n",
        "    output, h = net(inputs, h)\n",
        "    \n",
        "    # calculate loss\n",
        "    test_loss = criterion(output.squeeze(), labels.float())\n",
        "    test_losses.append(test_loss.item())\n",
        "    \n",
        "    # convert output probabilities to predicted class (0 or 1)\n",
        "    pred = torch.round(output.squeeze())  # rounds to the nearest integer\n",
        "    \n",
        "    # compare predictions to true label\n",
        "    correct_tensor = pred.eq(labels.float().view_as(pred))\n",
        "    correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())\n",
        "    num_correct += np.sum(correct)\n",
        "\n",
        "\n",
        "# -- stats! -- ##\n",
        "# avg test loss\n",
        "print(\"Test loss: {:.3f}\".format(np.mean(test_losses)))\n",
        "\n",
        "# accuracy over all test data\n",
        "test_acc = num_correct/len(test_loader.dataset)\n",
        "print(\"Test accuracy: {:.3f}\".format(test_acc))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test loss: 0.533\n",
            "Test accuracy: 0.795\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7MHF_RLtya-M",
        "colab_type": "text"
      },
      "source": [
        "# Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uuqLGyUIyXD2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# negative test review\n",
        "test_review_neg = 'The worst movie I have seen; acting was terrible and I want my money back. This movie had bad acting and the dialogue was slow.'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nbhJgAZIycst",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "35dccffb-cb81-42c9-bb91-b61b0284d0c1"
      },
      "source": [
        "from string import punctuation\n",
        "\n",
        "def tokenize_review(test_review):\n",
        "    test_review = test_review.lower() # lowercase\n",
        "    # get rid of punctuation\n",
        "    test_text = ''.join([c for c in test_review if c not in punctuation])\n",
        "\n",
        "    # splitting by spaces\n",
        "    test_words = test_text.split()\n",
        "\n",
        "    # tokens\n",
        "    test_ints = []\n",
        "    test_ints.append([vocab_to_int[word] for word in test_words])\n",
        "\n",
        "    return test_ints\n",
        "\n",
        "# test code and generate tokenized review\n",
        "test_ints = tokenize_review(test_review_neg)\n",
        "print(test_ints)\n",
        "\n",
        "# test sequence padding\n",
        "seq_length=200\n",
        "features = pad_features(test_ints, seq_length)\n",
        "\n",
        "print(features)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[1, 247, 18, 10, 28, 108, 113, 14, 388, 2, 10, 181, 60, 273, 144, 11, 18, 68, 76, 113, 2, 1, 410, 14, 539]]\n",
            "[[  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0   0\n",
            "    0   0   0   0   0   0   0   0   0   0   0   0   0   1 247  18  10  28\n",
            "  108 113  14 388   2  10 181  60 273 144  11  18  68  76 113   2   1 410\n",
            "   14 539]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r3ksc4TsyhbD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "161e23f1-509a-4023-cd34-2d12dae41eea"
      },
      "source": [
        "# test conversion to tensor and pass into the model\n",
        "feature_tensor = torch.from_numpy(features)\n",
        "print(feature_tensor.size())"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 200])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bvkGgJHTyk3p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict(net, test_review, sequence_length=200):\n",
        "    \n",
        "    net.eval()\n",
        "    \n",
        "    # tokenize review\n",
        "    test_ints = tokenize_review(test_review)\n",
        "    \n",
        "    # pad tokenized sequence\n",
        "    seq_length=sequence_length\n",
        "    features = pad_features(test_ints, seq_length)\n",
        "    \n",
        "    # convert to tensor to pass into your model\n",
        "    feature_tensor = torch.from_numpy(features)\n",
        "    \n",
        "    batch_size = feature_tensor.size(0)\n",
        "    \n",
        "    # initialize hidden state\n",
        "    h = net.init_hidden(batch_size)\n",
        "    \n",
        "    if(train_on_gpu):\n",
        "        feature_tensor = feature_tensor.cuda()\n",
        "    \n",
        "    # get the output from the model\n",
        "    output, h = net(feature_tensor, h)\n",
        "    \n",
        "    # convert output probabilities to predicted class (0 or 1)\n",
        "    pred = torch.round(output.squeeze()) \n",
        "    # printing output value, before rounding\n",
        "    print('Prediction value, pre-rounding: {:.6f}'.format(output.item()))\n",
        "    \n",
        "    # print custom response\n",
        "    if(pred.item()==1):\n",
        "        print(\"Positive review detected!\")\n",
        "    else:\n",
        "        print(\"Negative review detected.\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZNiCtyr4ypt-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "9d148fdd-7d45-4ead-d762-9cb0bf825ba7"
      },
      "source": [
        "# call function\n",
        "seq_length=200 # good to use the length that was trained on\n",
        "\n",
        "predict(net, test_review_neg, seq_length)\n"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Prediction value, pre-rounding: 0.007052\n",
            "Negative review detected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wzdUfqEPytdK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}