ML_Project_Extractive_Text_summarizer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import neural_network\n",
        "from sklearn import  metrics\n",
        "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
        "from keras.models import Sequential\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import keras\n",
        "import re\n",
        "import nltk"
      ],
      "metadata": {
        "id": "aMr9kgUdEjm5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uH5k4BwhIeJX",
        "outputId": "3e990467-cfb8-450c-88de-40197822409d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "english_stopwords = stopwords.words('english')\n",
        "english_stopwords.extend([\",\",\".\",\"$\",\"%\",\"'s\",\"``\",\"''\"])"
      ],
      "metadata": {
        "id": "YdQsRyZHIxKN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pattern = r'(?<=[.?!])(?:\\s*(?=[^0-9.]|[0-9]\\.[0-9]))'"
      ],
      "metadata": {
        "id": "ypySqW5fLHJX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_path = \"/content/drive/MyDrive/ML Folder/textsum_dataset/featuresDataset3.csv\""
      ],
      "metadata": {
        "id": "i1lsFY65EXZ1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_path = \"/content/drive/MyDrive/ML Folder/textsum_dataset/summarizer_model.joblib\""
      ],
      "metadata": {
        "id": "NBY9oGxHTDVG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(data_path)"
      ],
      "metadata": {
        "id": "K7qYQvn0WLJO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.drop(columns=[\"Unnamed: 0\"])"
      ],
      "metadata": {
        "id": "h5qwpJjlFj_3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "uQbQwwiPWU_w",
        "outputId": "24b1fb8c-6179-4d46-cee3-fa17ae1567b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   FisrtPara  FirstSente   ParaLoc  SenteLoc  SenteLen  ThematicWords  \\\n",
              "0          1           1  0.628319  3.141593        21       0.746269   \n",
              "1          0           1  0.339482  1.047198        23       2.158273   \n",
              "2          0           0  0.339482  0.565803        11       6.250000   \n",
              "3          0           0  0.339482  0.435788        20       1.941748   \n",
              "4          0           1  0.261473  0.523599        13       1.449275   \n",
              "\n",
              "   ProperNouns  StatRatio  Included  \n",
              "0     2.962963   2.222222         1  \n",
              "1     0.000000   2.112676         0  \n",
              "2     2.941176   0.000000         1  \n",
              "3     3.809524   0.000000         1  \n",
              "4    11.428571   0.000000         0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-aafa4452-832d-426c-b9dc-c037a3a01031\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FisrtPara</th>\n",
              "      <th>FirstSente</th>\n",
              "      <th>ParaLoc</th>\n",
              "      <th>SenteLoc</th>\n",
              "      <th>SenteLen</th>\n",
              "      <th>ThematicWords</th>\n",
              "      <th>ProperNouns</th>\n",
              "      <th>StatRatio</th>\n",
              "      <th>Included</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.628319</td>\n",
              "      <td>3.141593</td>\n",
              "      <td>21</td>\n",
              "      <td>0.746269</td>\n",
              "      <td>2.962963</td>\n",
              "      <td>2.222222</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.339482</td>\n",
              "      <td>1.047198</td>\n",
              "      <td>23</td>\n",
              "      <td>2.158273</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.112676</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.339482</td>\n",
              "      <td>0.565803</td>\n",
              "      <td>11</td>\n",
              "      <td>6.250000</td>\n",
              "      <td>2.941176</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.339482</td>\n",
              "      <td>0.435788</td>\n",
              "      <td>20</td>\n",
              "      <td>1.941748</td>\n",
              "      <td>3.809524</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.261473</td>\n",
              "      <td>0.523599</td>\n",
              "      <td>13</td>\n",
              "      <td>1.449275</td>\n",
              "      <td>11.428571</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-aafa4452-832d-426c-b9dc-c037a3a01031')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-aafa4452-832d-426c-b9dc-c037a3a01031 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-aafa4452-832d-426c-b9dc-c037a3a01031');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b3bc176d-f60e-4258-a670-7f3142d74b4f\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b3bc176d-f60e-4258-a670-7f3142d74b4f')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b3bc176d-f60e-4258-a670-7f3142d74b4f button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 43217,\n  \"fields\": [\n    {\n      \"column\": \"FisrtPara\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"FirstSente\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ParaLoc\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.2398094060369131,\n        \"min\": 0.0001853832038168,\n        \"max\": 1.5707963267948966,\n        \"num_unique_values\": 772,\n        \"samples\": [\n          0.1579601536439196,\n          0.0596657491514558\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SenteLoc\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.7771357392406482,\n        \"min\": 0.0005561496114506,\n        \"max\": 3.141592653589793,\n        \"num_unique_values\": 350,\n        \"samples\": [\n          0.0285688648318879,\n          0.0226090863260018\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SenteLen\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8,\n        \"min\": 1,\n        \"max\": 356,\n        \"num_unique_values\": 75,\n        \"samples\": [\n          13,\n          102\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ThematicWords\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 1.2785965858478732,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 916,\n        \"samples\": [\n          1.680672268907563,\n          3.045685279187817\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ProperNouns\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.966469965952806,\n        \"min\": 0.0,\n        \"max\": 57.14285714285714,\n        \"num_unique_values\": 1359,\n        \"samples\": [\n          6.535947712418301,\n          3.90625\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"StatRatio\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.3847207833213977,\n        \"min\": 0.0,\n        \"max\": 60.0,\n        \"num_unique_values\": 581,\n        \"samples\": [\n          4.245283018867925,\n          1.7341040462427746\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Included\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 1,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          0,\n          1\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X = df.iloc[:,:-1]\n",
        "Y = df.iloc[:, -1]\n",
        "print(Y.head())\n",
        "print(X.head())\n",
        "print(len(Y))\n",
        "print(len(X))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "igdyjrGfE99r",
        "outputId": "2662ea48-ef80-43d7-d072-ed14ba434b58"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0    1\n",
            "1    0\n",
            "2    1\n",
            "3    1\n",
            "4    0\n",
            "Name: Included, dtype: int64\n",
            "   FisrtPara  FirstSente   ParaLoc  SenteLoc  SenteLen  ThematicWords  \\\n",
            "0          1           1  0.628319  3.141593        21       0.746269   \n",
            "1          0           1  0.339482  1.047198        23       2.158273   \n",
            "2          0           0  0.339482  0.565803        11       6.250000   \n",
            "3          0           0  0.339482  0.435788        20       1.941748   \n",
            "4          0           1  0.261473  0.523599        13       1.449275   \n",
            "\n",
            "   ProperNouns  StatRatio  \n",
            "0     2.962963   2.222222  \n",
            "1     0.000000   2.112676  \n",
            "2     2.941176   0.000000  \n",
            "3     3.809524   0.000000  \n",
            "4    11.428571   0.000000  \n",
            "43217\n",
            "43217\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.20, random_state=42)"
      ],
      "metadata": {
        "id": "EP5Pg-ISE4Nt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# model = neural_network.MLPClassifier(\n",
        "#     alpha= 0.001,\n",
        "#     hidden_layer_sizes= (100),\n",
        "#     solver= 'adam',\n",
        "#     random_state= 50)\n",
        "# model.fit(x_train, y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "id": "oBnCRZwFGudO",
        "outputId": "df48d403-7def-42f0-a0c5-c7e4740b533a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MLPClassifier(alpha=0.001, hidden_layer_sizes=100, random_state=50)"
            ],
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MLPClassifier(alpha=0.001, hidden_layer_sizes=100, random_state=50)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MLPClassifier</label><div class=\"sk-toggleable__content\"><pre>MLPClassifier(alpha=0.001, hidden_layer_sizes=100, random_state=50)</pre></div></div></div></div></div>"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from joblib import dump, load\n",
        "# dump(model, 'summarizer_model.joblib')\n",
        "model = load(model_path)"
      ],
      "metadata": {
        "id": "bRaP-St1ImQQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "predicted = model.predict(x_val)\n",
        "print(\"Classification Report:\\n %s:\" % (metrics.classification_report(y_val, predicted)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V0YGmZsQGvZ0",
        "outputId": "1a0fe888-a575-4546-9dbb-2e38bf8903bf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Classification Report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.74      0.86      0.79      5370\n",
            "           1       0.69      0.49      0.57      3274\n",
            "\n",
            "    accuracy                           0.72      8644\n",
            "   macro avg       0.71      0.68      0.68      8644\n",
            "weighted avg       0.72      0.72      0.71      8644\n",
            ":\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def extract_features(article):\n",
        "  X=[]\n",
        "  pattern = r'(?<=[.?!])(?:\\s*(?=[^0-9.]|[0-9]\\.[0-9]))'\n",
        "  allWords=nltk.tokenize.word_tokenize(article)\n",
        "  allWordExceptStopDist = nltk.FreqDist(w.lower() for w in allWords if w.lower() not in english_stopwords and w.isalnum())\n",
        "  mostCommon= [k for k,c in allWordExceptStopDist.most_common(10)]\n",
        "\n",
        "  pos_tags = nltk.pos_tag(allWords)\n",
        "  proper_nouns = [word for word, pos_tag in pos_tags if pos_tag == 'NNP']\n",
        "  stats = [item for item, pos_tag in pos_tags if pos_tag in ['CD']]\n",
        "\n",
        "  articleFeatureVects=[]\n",
        "  yVect=[]\n",
        "  for j,para in enumerate(article.split(\"\\n\\n\")):\n",
        "      for k,sente in enumerate([sentence.rstrip('.?!') for sentence in re.split(pattern,para)]):\n",
        "          if len(sente)==0:\n",
        "              continue\n",
        "          senteFeatureVect=[]\n",
        "          senteFeatureVect.append(1 if j==0 else 0)   # eliminate this feature\n",
        "          senteFeatureVect.append(1 if k==0 else 0)\n",
        "          senteFeatureVect.append(np.absolute(np.pi*np.cos(j)/len(article.split(\"\\n\\n\"))))\n",
        "          senteFeatureVect.append(np.absolute(np.pi*np.cos(k)/len(para.split(\". \"))))\n",
        "          senteFeatureVect.append(len(sente.split(\" \")))\n",
        "\n",
        "          thematicWords=0\n",
        "          propnounWords=0\n",
        "          statsWords=0\n",
        "          for word in sente.split(\" \"):\n",
        "              if word in mostCommon:\n",
        "                  thematicWords+=1\n",
        "              if word in proper_nouns:\n",
        "                  propnounWords+=1\n",
        "              if word in stats:\n",
        "                  statsWords+=1\n",
        "          thematicWords=100*thematicWords/(len(sente)-thematicWords+1 if len(sente)-thematicWords==0 else len(sente)-thematicWords)\n",
        "          propnounWords=propnounWords/len(sente)*200\n",
        "          statsWords=statsWords/len(sente)*300\n",
        "\n",
        "          senteFeatureVect.append(thematicWords)\n",
        "          senteFeatureVect.append(propnounWords)\n",
        "          senteFeatureVect.append(statsWords)\n",
        "\n",
        "          articleFeatureVects.append(senteFeatureVect)\n",
        "      # break\n",
        "\n",
        "  X.extend(articleFeatureVects)\n",
        "  return (X)"
      ],
      "metadata": {
        "id": "Jullg5EtG-QI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "article = \"\"\"\n",
        "Modi said the people of Rajasthan had taught Congress a lesson in the first phase. “In the first phase, half of Rajasthan has punished the Congress and taught it a lesson. Full of patriotism, Rajasthan knows that Congress cannot ever make a strong Bharat. The country doesn’t want a Congress government. The country doesn’t want the pre-2014 situation to return. Everyone used to threaten the weak Congress government and everyone was busy looting the country. No one even used to ask for the Prime Minister, and the government used to be run by remote control,” he said.\n",
        "\n",
        "Highlighting his government’s achievements, particularly for Rajasthan, Modi said, around 19 lakh poor, Dalit, backward, and tribal families in the state have been given pucca houses, of which 1 lakh homes have been constructed in Jalore and Sirohi.\n",
        "\n",
        "The Congress has fielded Vaibhav Gehlot, son of former chief minister Ashok Gehlot, against BJP’s Lumbaram Choudhary from Jalore. The BJP’s Devji Patel is the sitting MP from the seat.\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "J5xhhWhxITEe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "article2 = \"\"\"\n",
        "In the early hours of Friday, Mehrdad, an engineer in Isfahan, Iran, woke to the sound of explosions rattling the windows and shaking the ground. In Tehran, passengers about to board flights were abruptly told the airspace was closed.\n",
        "\n",
        "Israel, they soon learned, had attacked Iran.\n",
        "\n",
        "As booms and gunfire went off in the distance, Mehrdad, 43, came to realize that the Israelis’ target was a military base on the outskirts of the city. He and his pregnant wife remained fearful that war would break out, he said in an interview by phone.\n",
        "\n",
        "“I think Israel wanted to test the water and evaluate with last night’s strikes,” said Mehrdad, who, like others interviewed for this article, asked that his last name be withheld for fear of retribution. “I fear the worst is coming, but I also hope that things end here.”\n",
        "\n",
        "So, apparently, does the Iranian government, which after a week of promising a forceful response to any Israeli attack on Iranian territory, appeared to be standing down from nearly going to the brink of war with Israel. Facing deep economic troubles and a restive population, the government seems to have adopted a two-track policy, analysts say, declaring victory over Israel and cracking down at home.\n",
        "\n",
        "“The external and internal challenges are two sides of the same coin for the establishment,” Abbas Abdi, a prominent analyst and writer in Tehran, said in a telephone interview. “With both Israel and internal dissent, they are taking an aggressive approach because they think both issues have reached a boiling point where if they do nothing it will only get worse.”\n",
        "\n",
        "The tit-for-tat attacks between Iran and Israel over the past three weeks were a startling and worrisome departure from the shadow warfare they have waged for decades, raising fears of a regional war. Iran responded to a deadly Israeli attack on its embassy compound in Damascus, Syria, by launching a barrage of more than 300 drones and missiles directly at Israel for the first time. A majority of them were intercepted.\n",
        "\n",
        "World leaders implored Israel to respond with restraint, which it did on Friday, attacking an Iranian air force base with drones. The strike damaged the radar of an S-300 system responsible for the air defense of Natanz nuclear facility in central Iran. Israel also fired air-to-ground missiles toward Iran but deliberately inflicted little damage. Afterward, Iranian state news media and officials downplayed the attack.\n",
        "\n",
        "Editors’ Picks\n",
        "\n",
        "Atlas, a Humanoid Robot From Boston Dynamics, Is Leaping Into Retirement\n",
        "\n",
        "Watch the Lyrid Meteor Shower Reach Its Peak\n",
        "\n",
        "It’s 4/20. These Restaurants Know You Have the Munchies.\n",
        "Image\n",
        "People on a street walk past a large billboard depicting missiles.\n",
        "Iran and Israel have waged shadow warfare for decades, but in recent weeks the conflict has escalated, raising fears of a regional war.Credit...Arash Khamooshi for The New York Times\n",
        "Nasser Imani, an analyst in Tehran with close ties to the government, said Iran had dealt effectively with Israel and could now afford to de-escalate.\n",
        "\n",
        "“Iranian officials do not want war with Israel,” he said in a telephone interview. “Iran will end it here and not directly engage any more because they feel they have established enough deterrence for now.”\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "_ecab7moiBs5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "article3 = \"\"\"\n",
        "India's power consumption grew nearly 10% year-on-year to 70.66 billion units (BU) in the first half of April this year, showing improvement in economic activities and consumption patterns, according to the power ministry data.\n",
        "\n",
        "According to the data, power consumption in the country rose to 70.66 BU during April 1-15 this year from 64.24 MU in the year-ago period.\n",
        "\n",
        "\n",
        "The peak power demand met or the highest supply in a day rose to about 218 GW in the first half of April compared to 206 GW in the same period a year ago.\n",
        "\n",
        "\n",
        "The highest supply in a day during the entire month of April last year was about 216 GW.\n",
        "\n",
        "The ministry has projected a peak power demand of 260 GW during the summer season (April to June) in view of longer heat wave duration. The peak power demand had touched an all-time high of 243 GW in September 2023.\n",
        "\n",
        "The India Meteorological Department (IMD) has predicted above-normal maximum temperatures in most parts of the country during summer this year.\n",
        "\n",
        "\n",
        "The experts said that the power demand will increase in the coming days as rains in different parts of the country have reduced the need for cooling appliances like air conditioners, desert coolers etc used during this time of the year.\n",
        "\n",
        "However, they said that power consumption growth in double digits shows improvement in economic activities and change in consumption patterns.\n",
        "\n",
        "They are of the view that the consumers in India are also increasing their energy consumption on various appliances and gadgets as is being done in the developed world.\n",
        "\n",
        "Besides, they pointed out that the increase in electricity in the transport sector like electric buses, cars, rickshaws and railways has also changed the consumption pattern and increased per capita use.\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "IVsGm3Rkrze9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def count_total_words(text):\n",
        "    # Split the text into words\n",
        "    words = text.split()\n",
        "\n",
        "    # Count the total number of words\n",
        "    total_words = len(words)\n",
        "\n",
        "    return total_words"
      ],
      "metadata": {
        "id": "2Oe0dRPag0tq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def summarize(article, max_sents=1000):\n",
        "  import warnings\n",
        "  warnings.filterwarnings('ignore')\n",
        "  features = extract_features(article)\n",
        "  article_sentences = []\n",
        "  for j,para in enumerate(article.split(\"\\n\\n\")):\n",
        "    for k,sente in enumerate([sentence.rstrip('.?!') for sentence in re.split(pattern,para)]):\n",
        "      if len(sente)==0:\n",
        "            continue\n",
        "      article_sentences.append(sente)\n",
        "  total_words = count_total_words(article)\n",
        "  summary_array = []\n",
        "  summarized_sentences = model.predict(features)\n",
        "  for i in range(len(summarized_sentences)):\n",
        "    if(summarized_sentences[i]):\n",
        "      summary_array.append(article_sentences[i])\n",
        "  summary=\"\\n\".join(summary_array[:min(len(summary_array),max_sents)])\n",
        "  total_summary_words = count_total_words(summary)\n",
        "  percent_compression = (total_words-total_summary_words)/total_words*100\n",
        "  return (summary, total_words, total_summary_words, percent_compression)"
      ],
      "metadata": {
        "id": "6RcDHXwLa6ZS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_summary(summary_content):\n",
        "  summary, total_words, total_summary_words, percent_compression = summary_content\n",
        "  print(\"Summarized text: \", summary)\n",
        "  print(\"Total words in the article: \", total_words)\n",
        "  print(\"Total words in the summary: \", total_summary_words)\n",
        "  print(\"Percentage compressed: \", percent_compression)"
      ],
      "metadata": {
        "id": "sB1AZFOLcLgn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print_summary(summarize(article2))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Llkvj2T-gSEe",
        "outputId": "6f274e67-f189-481a-eccd-2b3d9f090f29"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Summarized text:  He and his pregnant wife remained fearful that war would break out, he said in an interview by phone\n",
            "So, apparently, does the Iranian government, which after a week of promising a forceful response to any Israeli attack on Iranian territory, appeared to be standing down from nearly going to the brink of war with Israel\n",
            "“The external and internal challenges are two sides of the same coin for the establishment,” Abbas Abdi, a prominent analyst and writer in Tehran, said in a telephone interview\n",
            "“Iranian officials do not want war with Israel,” he said in a telephone interview\n",
            "Total words in the article:  533\n",
            "Total words in the summary:  99\n",
            "Percentage compressed:  81.42589118198875\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print_summary(summarize(article3))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oR5M5Pk_snj1",
        "outputId": "12619bed-32be-466f-d9f6-d112b01f2e49"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Summarized text:  \n",
            "India's power consumption grew nearly 10% year-on-year to 70.66 billion units (BU) in the first half of April this year, showing improvement in economic activities and consumption patterns, according to the power ministry data\n",
            "According to the data, power consumption in the country rose to 70.66 BU during April 1-15 this year from 64.24 MU in the year-ago period\n",
            "\n",
            "The peak power demand met or the highest supply in a day rose to about 218 GW in the first half of April compared to 206 GW in the same period a year ago\n",
            "The ministry has projected a peak power demand of 260 GW during the summer season (April to June) in view of longer heat wave duration\n",
            "The peak power demand had touched an all-time high of 243 GW in September 2023\n",
            "The India Meteorological Department (IMD) has predicted above-normal maximum temperatures in most parts of the country during summer this year\n",
            "\n",
            "The experts said that the power demand will increase in the coming days as rains in different parts of the country have reduced the need for cooling appliances like air conditioners, desert coolers etc used during this time of the year\n",
            "Total words in the article:  292\n",
            "Total words in the summary:  194\n",
            "Percentage compressed:  33.56164383561644\n"
          ]
        }
      ]
    }
  ]
}