Assignment_2_17110021_Anubhav_Jain.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Assignment-2-17110021-Anubhav_Jain.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yLe1n79DD7Xk",
        "colab_type": "text"
      },
      "source": [
        "# Neural Approach"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CuRad8CScu8p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.tokenize import sent_tokenize\n",
        "import string\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from nltk.tokenize import word_tokenize\n",
        "import random\n",
        "import re\n",
        "import math\n",
        "import nltk\n",
        "\n",
        "from keras.utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.layers import LSTM\n",
        "from keras.layers import Embedding"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3rRTm1CFzpzN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fp = open(\"/content/drive/My Drive/Theory_of_Computation/speeches.txt\", 'r', encoding=\"utf-8-sig\", errors='ignore')\n",
        "data = fp.read()\n",
        "\n",
        "data = data.replace(\"\\n\", \"\")\n",
        "data = data.replace(\"...\", \". \")\n",
        "data = re.sub(r\"SPEECH [0-9]\",\"\", data)\n",
        "data = re.sub(r\"[:;]\",\".\",data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbSDz5I90qBW",
        "colab_type": "code",
        "outputId": "bca92fda-1cdb-4b3b-fabf-561890fc97e3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5CsjyJK21eyK",
        "colab_type": "code",
        "outputId": "fb4233b5-f33d-476f-fcfb-45c1883f8992",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ruWTvV_XzsRZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sent_tokenize_list = sent_tokenize(data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_1IUa1MzrWq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lines = list()\n",
        "for i in range(len(sent_tokenize_list)):\n",
        "  if(len(sent_tokenize_list[i])>1):\n",
        "    lines.append(\"<s> \" + sent_tokenize_list[i][:-1] + \" </s>\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sh4u0LgUzzOX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "random.shuffle(lines)\n",
        "cut = int(0.8*len(lines))\n",
        "train = np.array(lines[:cut])\n",
        "test = np.array(lines[cut:])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_BLpcCDxz50e",
        "colab_type": "code",
        "outputId": "32042015-005f-43a0-c0a5-525c41c0c71e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "train"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['<s> Oh boy </s>',\n",
              "       '<s> Somebody else would come, they’d have like 50 people and they wouldn’t need this room </s>',\n",
              "       '<s> Where’s Sharon </s>', ...,\n",
              "       '<s> He borrowed a million dollars at an interest rate that everybody in this room would be proud to have – a very low interest rate </s>',\n",
              "       '<s> We love you </s>', '<s> But beautiful, beautiful bikes </s>'],\n",
              "      dtype='<U1761')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_ZaUZuae1ntx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def clean_doc(doc):\n",
        "  table = str.maketrans('', '', string.punctuation)\n",
        "  text = doc.translate(table)\n",
        "  tokens = word_tokenize(text)\n",
        "  tokens = [word for word in tokens if word.isalpha()]\n",
        "  tokens = [word.lower() for word in tokens]\n",
        "  return tokens"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ez_tk4q99DwN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences = list()\n",
        "seq_length = 2\n",
        "\n",
        "for i in range(len(train)):\n",
        "#   tokens = [\"<start>\"]\n",
        "  tokens = (clean_doc(train[i]))\n",
        "#   tokens.append(\"<end>\")\n",
        "  if(len(tokens)>seq_length):\n",
        "    for k in range(0,len(tokens)-seq_length):\n",
        "      sequences.append(tokens[k:k+seq_length+1])\n",
        "#   titles[i] = ' '.join(clean_doc(titles[i]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qly5V0Su9Zbf",
        "colab_type": "code",
        "outputId": "9165dcd3-4670-4a26-99af-0af7dbff88fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "sequences[:10]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['s', 'oh', 'boy'],\n",
              " ['oh', 'boy', 's'],\n",
              " ['s', 'somebody', 'else'],\n",
              " ['somebody', 'else', 'would'],\n",
              " ['else', 'would', 'come'],\n",
              " ['would', 'come', 'they'],\n",
              " ['come', 'they', 'd'],\n",
              " ['they', 'd', 'have'],\n",
              " ['d', 'have', 'like'],\n",
              " ['have', 'like', 'people']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nS4-AZIE9dMu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# integer encode sequences of words\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(sequences)\n",
        "sequences_intEncoded = tokenizer.texts_to_sequences(sequences)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3KoCQoUD9fIa",
        "colab_type": "code",
        "outputId": "696f94bd-e37c-4eb0-c75f-ced1b7c783b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# vocabulary size\n",
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "print(vocab_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7494\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PboGcvaQ9g-B",
        "colab_type": "code",
        "outputId": "6955552a-c7ed-48f5-c352-16e296b79287",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "sequences_intEncoded[:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[[1, 178, 864], [178, 864, 1], [1, 193, 269], [193, 269, 90], [269, 90, 112]]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J0MYlZk69jXW",
        "colab_type": "code",
        "outputId": "048e08b3-a112-4083-920a-1905658f1c77",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "sequences_intEncoded = np.array(sequences_intEncoded)\n",
        "np.shape(sequences)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(134740, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14r-TiYp9lxe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = sequences_intEncoded[:,:-1]\n",
        "y = sequences_intEncoded[:,-1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6huj7i8N9oUQ",
        "colab_type": "code",
        "outputId": "d62030b8-ceae-43d8-ca3e-ad84fa273d75",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "X[:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[  1, 178],\n",
              "       [178, 864],\n",
              "       [  1, 193],\n",
              "       [193, 269],\n",
              "       [269,  90]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oc7U-ilq9qg6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = to_categorical(y, num_classes=vocab_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VQqQf-dFIlds",
        "colab_type": "code",
        "outputId": "1f66711d-90b7-4968-c6be-0246d0aed845",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "###### VANILLA RECURRENT NEURAL NETWORK MODEL ######\n",
        "from keras.layers import SimpleRNN\n",
        "\n",
        "model1 = Sequential()\n",
        "model1.add(Embedding(vocab_size, 50, input_length=seq_length))\n",
        "model1.add(SimpleRNN(25, return_sequences=True))\n",
        "model1.add(SimpleRNN(25))\n",
        "model1.add(Dense(50, activation='relu'))\n",
        "model1.add(Dense(vocab_size, activation='softmax'))\n",
        "\n",
        "model1.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_4\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_3 (Embedding)      (None, 2, 50)             374700    \n",
            "_________________________________________________________________\n",
            "simple_rnn_3 (SimpleRNN)     (None, 2, 25)             1900      \n",
            "_________________________________________________________________\n",
            "simple_rnn_4 (SimpleRNN)     (None, 25)                1275      \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 50)                1300      \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 7494)              382194    \n",
            "=================================================================\n",
            "Total params: 761,369\n",
            "Trainable params: 761,369\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NNiaM08f9tsF",
        "colab_type": "code",
        "outputId": "6c68b3ce-eeeb-4033-ac51-d0d6515e1d2c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        }
      },
      "source": [
        "###### LONG SHORT TERM MEMORY NEURAL NETWORK MODEL ######\n",
        "model = Sequential()\n",
        "model.add(Embedding(vocab_size, 50, input_length=seq_length))\n",
        "model.add(LSTM(25, return_sequences=True))\n",
        "model.add(LSTM(25))\n",
        "model.add(Dense(50, activation='relu'))\n",
        "model.add(Dense(vocab_size, activation='softmax'))\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 2, 50)             375950    \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 2, 25)             7600      \n",
            "_________________________________________________________________\n",
            "lstm_2 (LSTM)                (None, 25)                5100      \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 50)                1300      \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 7519)              383469    \n",
            "=================================================================\n",
            "Total params: 773,419\n",
            "Trainable params: 773,419\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "znxqQqge9vh3",
        "colab_type": "code",
        "outputId": "7c565d6d-adc3-474d-dbe8-99ea006e34e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "###### TRAINING LONG SHORT TERM MEMORY NEURAL NETWORK MODEL ######\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.fit(X, y, batch_size=128, epochs=40)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "135608/135608 [==============================] - 63s 464us/step - loss: 5.8234 - acc: 0.1013\n",
            "Epoch 2/20\n",
            "135608/135608 [==============================] - 61s 447us/step - loss: 5.3923 - acc: 0.1419\n",
            "Epoch 3/20\n",
            "135608/135608 [==============================] - 61s 450us/step - loss: 5.0628 - acc: 0.1825\n",
            "Epoch 4/20\n",
            "135608/135608 [==============================] - 61s 448us/step - loss: 4.8499 - acc: 0.2016\n",
            "Epoch 5/20\n",
            "135608/135608 [==============================] - 61s 448us/step - loss: 4.6995 - acc: 0.2098\n",
            "Epoch 6/20\n",
            "135608/135608 [==============================] - 61s 447us/step - loss: 4.5879 - acc: 0.2172\n",
            "Epoch 7/20\n",
            "135608/135608 [==============================] - 60s 444us/step - loss: 4.4914 - acc: 0.2240\n",
            "Epoch 8/20\n",
            "135608/135608 [==============================] - 60s 444us/step - loss: 4.4036 - acc: 0.2290\n",
            "Epoch 9/20\n",
            "135608/135608 [==============================] - 60s 442us/step - loss: 4.3210 - acc: 0.2344\n",
            "Epoch 10/20\n",
            "135608/135608 [==============================] - 60s 443us/step - loss: 4.2426 - acc: 0.2407\n",
            "Epoch 11/20\n",
            "135608/135608 [==============================] - 60s 442us/step - loss: 4.1707 - acc: 0.2464\n",
            "Epoch 12/20\n",
            "135608/135608 [==============================] - 60s 440us/step - loss: 4.1023 - acc: 0.2526\n",
            "Epoch 13/20\n",
            "135608/135608 [==============================] - 60s 442us/step - loss: 4.0401 - acc: 0.2588\n",
            "Epoch 14/20\n",
            "135608/135608 [==============================] - 60s 441us/step - loss: 3.9809 - acc: 0.2633\n",
            "Epoch 15/20\n",
            "135608/135608 [==============================] - 60s 445us/step - loss: 3.9266 - acc: 0.2675\n",
            "Epoch 16/20\n",
            "135608/135608 [==============================] - 60s 442us/step - loss: 3.8748 - acc: 0.2711\n",
            "Epoch 17/20\n",
            "135608/135608 [==============================] - 60s 442us/step - loss: 3.8242 - acc: 0.2757\n",
            "Epoch 18/20\n",
            "135608/135608 [==============================] - 60s 445us/step - loss: 3.7783 - acc: 0.2794\n",
            "Epoch 19/20\n",
            "135608/135608 [==============================] - 60s 443us/step - loss: 3.7329 - acc: 0.2839\n",
            "Epoch 20/20\n",
            "135608/135608 [==============================] - 61s 447us/step - loss: 3.6914 - acc: 0.2873\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fdc83bdeb00>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cc5giR59-1q-",
        "colab_type": "code",
        "outputId": "ff37291d-58bb-43d2-a042-8e1d120e9bdd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "###### TRAINING VANILLA RECURRENT NEURAL NETWORK MODEL ######\n",
        "\n",
        "model1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model1.fit(X, y, batch_size=128, epochs=20)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "134740/134740 [==============================] - 57s 420us/step - loss: 5.7983 - acc: 0.1227\n",
            "Epoch 2/20\n",
            "134740/134740 [==============================] - 55s 405us/step - loss: 4.9856 - acc: 0.1879\n",
            "Epoch 3/20\n",
            "134740/134740 [==============================] - 54s 402us/step - loss: 4.6936 - acc: 0.2108\n",
            "Epoch 4/20\n",
            "134740/134740 [==============================] - 59s 435us/step - loss: 4.5000 - acc: 0.2233\n",
            "Epoch 5/20\n",
            "134740/134740 [==============================] - 55s 405us/step - loss: 4.3449 - acc: 0.2352\n",
            "Epoch 6/20\n",
            "134740/134740 [==============================] - 54s 397us/step - loss: 4.2119 - acc: 0.2443\n",
            "Epoch 7/20\n",
            "134740/134740 [==============================] - 55s 409us/step - loss: 4.0913 - acc: 0.2542\n",
            "Epoch 8/20\n",
            "134740/134740 [==============================] - 55s 405us/step - loss: 3.9821 - acc: 0.2628\n",
            "Epoch 9/20\n",
            "134740/134740 [==============================] - 55s 412us/step - loss: 3.8814 - acc: 0.2715\n",
            "Epoch 10/20\n",
            "134740/134740 [==============================] - 55s 406us/step - loss: 3.7903 - acc: 0.2775\n",
            "Epoch 11/20\n",
            "134740/134740 [==============================] - 54s 403us/step - loss: 3.7079 - acc: 0.2852\n",
            "Epoch 12/20\n",
            "134740/134740 [==============================] - 54s 402us/step - loss: 3.6334 - acc: 0.2915\n",
            "Epoch 13/20\n",
            "134740/134740 [==============================] - 55s 405us/step - loss: 3.5692 - acc: 0.2978\n",
            "Epoch 14/20\n",
            "134740/134740 [==============================] - 54s 403us/step - loss: 3.5089 - acc: 0.3031\n",
            "Epoch 15/20\n",
            "134740/134740 [==============================] - 55s 406us/step - loss: 3.4590 - acc: 0.3093\n",
            "Epoch 16/20\n",
            "134740/134740 [==============================] - 55s 410us/step - loss: 3.4141 - acc: 0.3147\n",
            "Epoch 17/20\n",
            "134740/134740 [==============================] - 55s 409us/step - loss: 3.3736 - acc: 0.3203\n",
            "Epoch 18/20\n",
            "134740/134740 [==============================] - 54s 403us/step - loss: 3.3369 - acc: 0.3243\n",
            "Epoch 19/20\n",
            "134740/134740 [==============================] - 56s 415us/step - loss: 3.3057 - acc: 0.3289\n",
            "Epoch 20/20\n",
            "134740/134740 [==============================] - 57s 425us/step - loss: 3.2785 - acc: 0.3324\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fdc82486390>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5J5ADlReJnz_",
        "colab_type": "text"
      },
      "source": [
        "**Random text of 5 sentences**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UUSo_MugJ3iY",
        "colab_type": "code",
        "outputId": "323e7772-7d97-42d4-fc08-e7ab4f58282b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "###### LSTM OUTPUT WORDS ######\n",
        "\n",
        "total_score = 0\n",
        "total_count = 0\n",
        "generated_test_lstm = []\n",
        "count = 0\n",
        "\n",
        "for i in range(len(test)):\n",
        "  result = list()\n",
        "  text = clean_doc(test[i])\n",
        "  \n",
        "  if(len(text)>2):\n",
        "    result = text[:seq_length]\n",
        "    r = -1*seq_length\n",
        "    for _ in range(2,len(text)):\n",
        "      encoded = tokenizer.texts_to_sequences([' '.join(result[r:])])[0]\n",
        "      if(len(encoded)<seq_length):\n",
        "        for j in range(len(encoded),seq_length):\n",
        "          encoded.append(0)\n",
        "      encoded = np.array([encoded])\n",
        "      yhat = model.predict_classes(encoded, verbose=0)\n",
        "      out_word = ''\n",
        "      for word, index in tokenizer.word_index.items():\n",
        "        if index == yhat:\n",
        "          out_word = word\n",
        "          break\n",
        "      result.append(out_word)\n",
        "    \n",
        "    generated_test_lstm.append(result)\n",
        "    #print(test[i])\n",
        "\n",
        "    if(count < 5): ### to print 5 sentences only\n",
        "      print(result)\n",
        "    count+=1"
      ],
      "execution_count": 127,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['s', 'so', 'going', 'to', 'he', 'a', 'on', 'china', 's']\n",
            "['s', 'thats', 'about', 'some']\n",
            "['s', 'thats', 'about', 'some', 'be', 's', 'a', 'now', 'of', 'be']\n",
            "['s', 'ive', 'is', 'what', 's', 'going', 'to', 'he', 'a', 'on', 'china', 's', 'a']\n",
            "['s', 'all', 'going', 'to']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sX80AE6ICD7e",
        "outputId": "4221ff8f-6c9b-425b-b476-857f337204d7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "##### VANILLA RNN OUTPUT WORDS #######\n",
        "\n",
        "total_score = 0\n",
        "total_count = 0\n",
        "generated_test_rnn = []\n",
        "count = 0\n",
        "\n",
        "for i in range(len(test)):\n",
        "  result = list()\n",
        "  text = clean_doc(test[i])\n",
        "  \n",
        "  if(len(text)>2):\n",
        "    result = text[:seq_length]\n",
        "    r = -1*seq_length\n",
        "    for _ in range(2,len(text)):\n",
        "      encoded = tokenizer.texts_to_sequences([' '.join(result[r:])])[0]\n",
        "      if(len(encoded)<seq_length):\n",
        "        for j in range(len(encoded),seq_length):\n",
        "          encoded.append(0)\n",
        "      encoded = np.array([encoded])\n",
        "      yhat = model1.predict_classes(encoded, verbose=0)\n",
        "      out_word = ''\n",
        "      for word, index in tokenizer.word_index.items():\n",
        "        if index == yhat:\n",
        "          out_word = word\n",
        "          break\n",
        "      result.append(out_word)\n",
        "    \n",
        "    generated_test_rnn.append(result)\n",
        "    #print(test[i])\n",
        "\n",
        "    if(count < 5): ### to print 5 sentences only\n",
        "      print(result)\n",
        "    count+=1"
      ],
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['s', 'so', 'i', 'think', 'it', 's', 'a', 'great', 'company']\n",
            "['s', 'thats', 'the', 'way']\n",
            "['s', 'thats', 'the', 'way', 's', 'historyso', 'many', 'people', 's', 'like']\n",
            "['s', 'ive', 'admired', 'the', 'work', 's', 'been', 'a', 'great', 'company', 's', 'a', 'great']\n",
            "['s', 'all', 'of', 'the']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OfLko1rPCXyu",
        "colab_type": "text"
      },
      "source": [
        "**Calculation of Perplexity**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bz_blshaBDfh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file1 = open(\"/content/drive/My Drive/Theory_of_Computation/speeches.txt\", 'r', encoding=\"utf-8-sig\", errors='ignore')\n",
        "train_1 = file1.read()\n",
        "train_1 = trainstr.lower()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eAphK3sPBEeX",
        "colab_type": "code",
        "outputId": "a0b8ca18-8753-4c04-dff4-26ef8c4adb64",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "t1 = sent_tokenize(train_1)\n",
        "for i in range (len(t1)):\n",
        "  t1[i] = re.sub(r'[^A-Za-z\\s\\']+', \"\", t1[i])\n",
        "  t1[i] = '<s> '+t1[i]+' </s>'\n",
        "train = t1[:13000] # 80% of dataset is train\n",
        "test = t1[13000:] # 20% of dataset is test\n",
        "print(len(t1))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16401\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mIw1iObKBFtj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import Counter\n",
        "fdist1 = {} #bigram\n",
        "for i in range (len(train)):\n",
        "  tokens = train[i].split()\n",
        "  bigrams = nltk.bigrams(tokens)\n",
        "  fdist = dict(nltk.FreqDist(bigrams))\n",
        "  fdist1 = dict(Counter(fdist)+Counter(fdist1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gxx7jto7BF6H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trainstr = re.sub(r'[^A-Za-z\\s\\']+', \"\", trainstr)\n",
        "ls = trainstr.split()\n",
        "d = {}\n",
        "for i in range (len(ls)):\n",
        "  if ls[i] not in d:\n",
        "    d[ls[i]] = 1\n",
        "  else:\n",
        "    d[ls[i]] += 1\n",
        "Vocab = len(d)\n",
        "Token = sum(d.values())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rG8p0TK4ACzM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MLEbigram(w1,w2):\n",
        "  if (w1+\" \"+w2) not in fdist1:\n",
        "    if w1 in d.keys():\n",
        "      return 1/(d[w1]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return 1/(Vocab) # Add 1 Smoothing\n",
        "  else:\n",
        "    if w1 in d.keys():\n",
        "      return (fdist1(w1+\" \"+w2)+1)/(d[w1]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return (fdist1(w1+\" \"+w2)+1)/(Vocab) # Add 1 Smoothing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eNtKuoRxAD77",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def bigramsentenceprob(sentence):\n",
        "  bigram_sentence_probability_log_sum = 0\n",
        "  previous_word = None\n",
        "  for word in sentence:\n",
        "    if previous_word!=None:\n",
        "      x = MLEbigram(previous_word,word)\n",
        "      bigram_sentence_probability_log_sum += math.log(x,2)\n",
        "    previous_word = word\n",
        "  return math.pow(2, bigram_sentence_probability_log_sum)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uPzOwQsLAMpJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_number_of_bigrams(sentences):\n",
        "  bigram_count = 0\n",
        "  for sentence in sentences:\n",
        "    bigram_count += len(sentence) - 1\n",
        "  return bigram_count"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KjAwVwb7_KOm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_bigram_perplexity(sentences):\n",
        "  bigram_count = calculate_number_of_bigrams(sentences)\n",
        "  sentence_probability_log_sum = 0\n",
        "  for sentence in sentences:\n",
        "    try:\n",
        "      sentence_probability_log_sum -= math.log(bigramsentenceprob(sentence), 2)\n",
        "    except:\n",
        "      sentence_probability_log_sum -= 0\n",
        "  return math.pow(2, sentence_probability_log_sum / bigram_count)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tV04CPokAlk2",
        "colab_type": "code",
        "outputId": "61fc2ea6-3501-43a1-ba5f-8a98051ad951",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(\"Perplexity of test corpus with respect to bigram model is:\",end = \" \")\n",
        "print(calculate_bigram_perplexity(test))\n",
        "\n",
        "print(\"Perplexity of test corpus generated by LSTM architecture with respect to bigram model is:\",end = \" \")\n",
        "print(calculate_bigram_perplexity(generated_test_lstm))"
      ],
      "execution_count": 129,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Perplexity of test corpus with respect to bigram model is: 147.40810983034604\n",
            "Perplexity of test corpus generated by LSTM architecture with respect to bigram model is: 7622.6088372200265\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wqhfA_tZDrK4",
        "colab_type": "code",
        "outputId": "d57d3de7-3d2a-49a3-99b6-9e88d595a5f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(\"Perplexity of test corpus with respect to bigram model is:\",end = \" \")\n",
        "print(calculate_bigram_perplexity(test))\n",
        "\n",
        "print(\"Perplexity of test corpus generated by Baseline RNN architecture with respect to bigram model is:\",end = \" \")\n",
        "print(calculate_bigram_perplexity(generated_test_rnn))"
      ],
      "execution_count": 130,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Perplexity of test corpus with respect to bigram model is: 147.40810983034604\n",
            "Perplexity of test corpus generated by Baseline RNN architecture with respect to bigram model is: 7361.914927290586\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uRZ7e8lxEFR5",
        "colab_type": "text"
      },
      "source": [
        "# Classical Approach"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qg_Gr2jE8WHS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.tokenize import sent_tokenize, word_tokenize \n",
        "from nltk.util import ngrams\n",
        "from collections import Counter\n",
        "import re\n",
        "import numpy as np\n",
        "import random\n",
        "import math\n",
        "import operator\n",
        "import string"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ARGbaQax8WPR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "file2 = open(\"/content/drive/My Drive/Theory_of_Computation/speeches.txt\", 'r', encoding=\"utf-8-sig\", errors='ignore')\n",
        "trainstr = file2.read()\n",
        "trainstr = trainstr.lower()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OEohihYO8WY7",
        "colab_type": "code",
        "outputId": "abfaad9a-bac1-48df-c339-76f95d1c3214",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tnew = sent_tokenize(trainstr)\n",
        "for i in range (len(tnew)):\n",
        "  tnew[i] = re.sub(r'[^A-Za-z\\s\\']+', \"\", tnew[i])\n",
        "  tnew[i] = '<s> '+tnew[i]+' </s>'\n",
        "train = tnew[:13000] # 80% of dataset is train\n",
        "test = tnew[13000:] # 20% of dataset is test\n",
        "print(len(tnew))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "16401\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9vwQwi538W3k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fdist1 = {} #bigram\n",
        "for i in range (len(train)):\n",
        "  tokens = train[i].split()\n",
        "  bigrams = nltk.bigrams(tokens)\n",
        "  fdist = dict(nltk.FreqDist(bigrams))\n",
        "  fdist1 = dict(Counter(fdist)+Counter(fdist1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2HsTMLBn8XjD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "fdist2 = {} #trigram\n",
        "for i in range (len(train)):\n",
        "  tokens = train[i].split()\n",
        "  trigrams = nltk.trigrams(tokens)\n",
        "  fdist = dict(nltk.FreqDist(trigrams))\n",
        "  fdist2 = dict(Counter(fdist)+Counter(fdist2))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1FYr_Udu8Xu5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk import everygrams\n",
        "fdist3 = {} #quadgram\n",
        "for i in range (len(train)):\n",
        "  tokens = train[i].split()\n",
        "  fourgrams = list(everygrams(tokens,4,4)) \n",
        "  fdist = dict(nltk.FreqDist(fourgrams))\n",
        "  fdist3 = dict(Counter(fdist)+Counter(fdist3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9BgSiufn8X6g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "trainstr = re.sub(r'[^A-Za-z\\s\\']+', \"\", trainstr)\n",
        "ls = trainstr.split()\n",
        "d = {}\n",
        "for i in range (len(ls)):\n",
        "  if ls[i] not in d:\n",
        "    d[ls[i]] = 1\n",
        "  else:\n",
        "    d[ls[i]] += 1\n",
        "Vocab = len(d)\n",
        "Token = sum(d.values())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G1g6UBzi9TZW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MLEunigram(w1):\n",
        "  if w1 not in d:\n",
        "    return 1/(Vocab+Token) # Add 1 Smoothing\n",
        "  else:\n",
        "    return (d[w1]+1)/(Vocab+Token) # Add 1 Smoothing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5MHXDJjq9U_B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MLEbigram(w1,w2):\n",
        "  if (w1+\" \"+w2) not in fdist1:\n",
        "    if w1 in d.keys():\n",
        "      return 1/(d[w1]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return 1/(Vocab) # Add 1 Smoothing\n",
        "  else:\n",
        "    if w1 in d.keys():\n",
        "      return (fdist1(w1+\" \"+w2)+1)/(d[w1]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return (fdist1(w1+\" \"+w2)+1)/(Vocab) # Add 1 Smoothing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jcDrdpVq9WtC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MLEtrigram(w1,w2,w3):\n",
        "  if (w1+\" \"+w2+\" \"+w3) not in fdist2:\n",
        "    if w1+\" \"+w2 in fdist1.keys():\n",
        "      return 1/(fdist1[w1+\" \"+w2]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return 1/(Vocab) # Add 1 Smoothing\n",
        "  else:\n",
        "    if w1+\" \"+w2 in fdist1.keys():\n",
        "      return (fdist2(w1+\" \"+w2+\" \"+w3)+1)/(fdist1[w1+\" \"+w2]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return (fdist2(w1+\" \"+w2+\" \"+w3)+1)/(Vocab) # Add 1 Smoothing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0W4Cm4nd9YfC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MLEquadgram(w1,w2,w3,w4):\n",
        "  if (w1+\" \"+w2+\" \"+w3+\" \"+w4) not in fdist3:\n",
        "    if w1+\" \"+w2 + \" \"+w3 in fdist2.keys():\n",
        "      return 1/(fdist2[w1+\" \"+w2+\" \"+w3]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return 1/(Vocab) # Add 1 Smoothing\n",
        "  else:\n",
        "    if w1+\" \"+w2 + \" \"+w3 in fdlist2.keys():\n",
        "      return (fdist3(w1+\" \"+w2+\" \"+w3+\" \"+w4)+1)/(fdist2[w1+\" \"+w2+\" \"+w3]+Vocab) # Add 1 Smoothing\n",
        "    else:\n",
        "      return (fdist3(w1+\" \"+w2+\" \"+w3+\" \"+w4)+1)/(Vocab) # Add 1 Smoothing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IJpFtrbk9acC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def unigramsentenceprob(sentence):\n",
        "  sentence_probability_log_sum = 0\n",
        "  for word in sentence:\n",
        "    x = MLEunigram(word)\n",
        "    sentence_probability_log_sum += math.log(x,2)\n",
        "  return math.pow(2, sentence_probability_log_sum)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gfXurj_d9cKf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def bigramsentenceprob(sentence):\n",
        "  bigram_sentence_probability_log_sum = 0\n",
        "  previous_word = None\n",
        "  for word in sentence:\n",
        "    if previous_word!=None:\n",
        "      x = MLEbigram(previous_word,word)\n",
        "      bigram_sentence_probability_log_sum += math.log(x,2)\n",
        "    previous_word = word\n",
        "  return math.pow(2, bigram_sentence_probability_log_sum)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0-vznvtP9d7o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def trigramsentenceprob(sentence):\n",
        "  trigram_sentence_probability_log_sum = 0\n",
        "  previous_word = None\n",
        "  previous_previous_word = None\n",
        "  for word in sentence:\n",
        "    if previous_word!=None and previous_previous_word!=None:\n",
        "      x = MLEtrigram(previous_previous_word,previous_word,word)\n",
        "      trigram_sentence_probability_log_sum += math.log(x,2)\n",
        "    previous_previous_word = previous_word\n",
        "    previous_word = word\n",
        "  return math.pow(2, trigram_sentence_probability_log_sum)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K3CfMNvL9gHw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def quadgramsentenceprob(sentence):\n",
        "  quadgram_sentence_probability_log_sum = 0\n",
        "  previous_word = None\n",
        "  previous_previous_word = None\n",
        "  previous_previous_previous_word = None\n",
        "  for word in sentence:\n",
        "    if previous_word!=None and previous_previous_word!=None and previous_previous_previous_word!=None :\n",
        "      x = MLEquadgram(previous_previous_previous_previous_previous_word,previous_word,word)\n",
        "      quadgram_sentence_probability_log_sum += math.log(x,2)\n",
        "    previous_previous_previous_word = previous_previous_word\n",
        "    previous_previous_word = previous_word\n",
        "    previous_word = word\n",
        "  return math.pow(2, quadgram_sentence_probability_log_sum)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WTKYMCgk9iCc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_number_of_unigrams(sentences):\n",
        "  unigram_count = 0\n",
        "  for sentence in sentences:\n",
        "    # remove two for <s> and </s>\n",
        "    unigram_count += len(sentence) - 2\n",
        "  return unigram_count\n",
        "def calculate_number_of_bigrams(sentences):\n",
        "  bigram_count = 0\n",
        "  for sentence in sentences:\n",
        "    bigram_count += len(sentence) - 1\n",
        "  return bigram_count\n",
        "def calculate_number_of_trigrams(sentences):\n",
        "  trigram_count = 0\n",
        "  for sentence in sentences:\n",
        "    trigram_count += len(sentence) - 2\n",
        "  return trigram_count\n",
        "def calculate_number_of_quadgrams(sentences):\n",
        "  quadgram_count = 0\n",
        "  for sentence in sentences:\n",
        "    quadgram_count += len(sentence) - 3\n",
        "  return quadgram_count"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2VyVDwr19j68",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_unigram_perplexity(sentences):\n",
        "  unigram_count = calculate_number_of_unigrams(sentences)\n",
        "  sentence_probability_log_sum = 0\n",
        "  for sentence in sentences:\n",
        "    try:\n",
        "      sentence_probability_log_sum -= math.log(unigramsentenceprob(sentence), 2)\n",
        "    except:\n",
        "      sentence_probability_log_sum -= 0\n",
        "  return math.pow(2, sentence_probability_log_sum / unigram_count)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pNkBPs4E9l8R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_bigram_perplexity(sentences):\n",
        "  bigram_count = calculate_number_of_bigrams(sentences)\n",
        "  sentence_probability_log_sum = 0\n",
        "  for sentence in sentences:\n",
        "    try:\n",
        "      sentence_probability_log_sum -= math.log(bigramsentenceprob(sentence), 2)\n",
        "    except:\n",
        "      sentence_probability_log_sum -= 0\n",
        "  return math.pow(2, sentence_probability_log_sum / bigram_count)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xcFMT6Po9n0d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calculate_trigram_perplexity(sentences):\n",
        "  trigram_count = calculate_number_of_bigrams(sentences)\n",
        "  sentence_probability_log_sum = 0\n",
        "  for sentence in sentences:\n",
        "    try:\n",
        "      sentence_probability_log_sum -= math.log(trigramsentenceprob(sentence), 2)\n",
        "    except:\n",
        "      sentence_probability_log_sum -= 0\n",
        "  return math.pow(2, sentence_probability_log_sum / trigram_count)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "d6d9iOHbJMOn",
        "colab": {}
      },
      "source": [
        "def calculate_quadgram_perplexity(sentences):\n",
        "  quadgram_count = calculate_number_of_quadgrams(sentences)\n",
        "  sentence_probability_log_sum = 0\n",
        "  for sentence in sentences:\n",
        "    try:\n",
        "      sentence_probability_log_sum -= math.log(quadgramsentenceprob(sentence), 2)\n",
        "    except:\n",
        "      sentence_probability_log_sum -= 0\n",
        "  return math.pow(2, sentence_probability_log_sum / quadgram_count)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IJ7kHZHuJOR5",
        "colab_type": "text"
      },
      "source": [
        "**Calculation of Perplexity**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rDAOgijI9rRw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "99ae05c0-d0d1-469c-a080-dd07415dfe01"
      },
      "source": [
        "import math\n",
        "print(\"Perplexity of test corpus with respect to unigram model is:\",end = \" \")\n",
        "print(calculate_unigram_perplexity(test))\n",
        "print(\"Perplexity of test corpus with respect to bigram model is:\",end = \" \")\n",
        "print(calculate_bigram_perplexity(test))\n",
        "print(\"Perplexity of test corpus with respect to trigram model is:\",end = \" \")\n",
        "print(calculate_trigram_perplexity(test))\n",
        "print(\"Perplexity of test corpus with respect to quadgram model is:\",end = \" \")\n",
        "print(calculate_quadgram_perplexity(test))\n"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Perplexity of test corpus with respect to unigram model is: 186.64101133266124\n",
            "Perplexity of test corpus with respect to bigram model is: 147.40810983034604\n",
            "Perplexity of test corpus with respect to trigram model is: 135.92047675682477\n",
            "Perplexity of test corpus with respect to quadgram model is: 1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zfeb1PgK9tZo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Classical generator to generate new words using a n_gram model\n",
        "def MLE_Generator(n_gram, initial_sequence):\n",
        "  sentence = [\"<s>\"]\n",
        "  if(n_gram==1):\n",
        "    for i in range(20):\n",
        "      max_prob = 0\n",
        "      max_prob_list = list()\n",
        "      for j in d.keys():\n",
        "        k = (MLEunigram(j))\n",
        "        if(k>max_prob):\n",
        "          max_prob = k\n",
        "          max_prob_list = [j]\n",
        "        elif(k==max_prob):\n",
        "          max_prob_list.append(j)\n",
        "      samples = np.random.multinomial(20,[max_prob]*len(max_prob_list),size=1)\n",
        "      index, value = max(enumerate(samples), key = operator.itemgetter(1))\n",
        "      sentence.append(max_prob_list[index])\n",
        "  else:\n",
        "    sentence.extend(initial_sequence)\n",
        "    i = len(initial_sequence)\n",
        "    while(sentence[-1]!=\"</s>\" and i<20):\n",
        "      max_prob = 0\n",
        "      max_prob_list = list()\n",
        "      for j in d.keys():\n",
        "        word_list = sentence[-n_gram+1:]\n",
        "        word_list.append(j)\n",
        "        if(n_gram==2):\n",
        "          k = MLEbigram(word_list[0],word_list[1])\n",
        "        elif(n_gram==3):\n",
        "          k = MLEtrigram(word_list[0],word_list[1],word_list[2])\n",
        "        elif(n_gram==4):\n",
        "          k = MLEquadgram(word_list[0],word_list[1],word_list[2],word_list[3])\n",
        "        if(k>max_prob):\n",
        "          max_prob = k\n",
        "          max_prob_list = [j]\n",
        "        elif(k==max_prob):\n",
        "          max_prob_list.append(j)\n",
        "      samples = np.random.multinomial(20,[max_prob]*len(max_prob_list),size=1)\n",
        "      index, value = max(enumerate(samples), key = operator.itemgetter(1))\n",
        "      sentence.append(max_prob_list[index])\n",
        "      i+=1\n",
        "    \n",
        "  return sentence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v3F6UszC9wso",
        "colab_type": "code",
        "outputId": "915015c4-99ba-483a-9468-5b049cf5efc7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(\" \".join(MLE_Generator(3,[\"it\", \"should\", \"not\", \"be\"])))\n"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<s> it should not be speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HO0zjY-BJT0o",
        "colab_type": "text"
      },
      "source": [
        "**Random text generation of 5 sentences**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phNxPLItFMAO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "e605ccfe-8136-4533-c3a7-d7da08d9b8ec"
      },
      "source": [
        "# GENERATING RANDOM TEXT:\n",
        "# UNIGRAMS\n",
        "print(\"UNIGRAM: \", end = \" \")\n",
        "print(\" \".join(MLE_Generator(1,[])))\n",
        "# BIGRAM\n",
        "print(\"BIGRAM: \", end = \" \")\n",
        "print(\" \".join(MLE_Generator(2,[\"it\", \"is\"])))\n",
        "# TRIGRAM\n",
        "print(\"TRIGRAM: \", end = \" \")\n",
        "print(\" \".join(MLE_Generator(3,[\"there\", \"is\", \"something\"])))\n",
        "# QUADGRAM\n",
        "print(\"QUADGRAM1: \", end = \" \")\n",
        "print(\" \".join(MLE_Generator(4,[\"but\", \"there\", \"is\", \"no\"])))\n",
        "# ANOTHER QuADGRAM\n",
        "print(\"QUADGRAM2: \", end = \" \")\n",
        "print(\" \".join(MLE_Generator(4,[\"it\", \"is\", \"to\", \"be\"])))"
      ],
      "execution_count": 126,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "UNIGRAM:  <s> the the the the the the the the the the the the the the the the the the the the\n",
            "BIGRAM:  <s> it is speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech\n",
            "TRIGRAM:  <s> there is something speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech\n",
            "QUADGRAM1:  <s> but there is no speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech\n",
            "QUADGRAM2:  <s> it is to be speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech speech\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}