ULMFiT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled",
      "provenance": [],
      "authorship_tag": "ABX9TyPz9yX2hlqcgTc5AyMtPZOF",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jzm-123/test/blob/master/ULMFiT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "3SKqn1Ngfzqh",
        "outputId": "86184eaa-2a0e-4015-843d-3cb17c8503ce"
      },
      "source": [
        "pip install fastai==2.1.5\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting fastai==2.1.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9a/da/99c526c11762d7876a26e08995f179594d9e13b5121d3789e938d1251f7c/fastai-2.1.5-py3-none-any.whl (188kB)\n",
            "\r\u001b[K     |█▊                              | 10kB 23.4MB/s eta 0:00:01\r\u001b[K     |███▌                            | 20kB 30.6MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 30kB 16.1MB/s eta 0:00:01\r\u001b[K     |███████                         | 40kB 12.2MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 51kB 11.4MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 61kB 10.4MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 71kB 10.2MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 81kB 10.9MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 92kB 11.0MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 102kB 10.8MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 112kB 10.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 122kB 10.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 133kB 10.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 143kB 10.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 153kB 10.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 163kB 10.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 174kB 10.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 184kB 10.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 194kB 10.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (1.4.1)\n",
            "Requirement already satisfied: torch>=1.7.0 in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (1.7.0+cu101)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (0.22.2.post1)\n",
            "Requirement already satisfied: pip in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (19.3.1)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (7.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (20.4)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (3.13)\n",
            "Requirement already satisfied: torchvision>=0.8 in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (0.8.1+cu101)\n",
            "Collecting fastcore>=1.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/59/80/a61e8b91e2f65b0b6fd1449008470545ad771d6041e529504c6e5df4d4e7/fastcore-1.3.6-py3-none-any.whl (48kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 7.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (3.2.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (2.23.0)\n",
            "Requirement already satisfied: fastprogress>=0.2.4 in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (1.0.0)\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (2.2.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from fastai==2.1.5) (1.1.4)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy->fastai==2.1.5) (1.18.5)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch>=1.7.0->fastai==2.1.5) (0.7)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch>=1.7.0->fastai==2.1.5) (3.7.4.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.7.0->fastai==2.1.5) (0.16.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->fastai==2.1.5) (0.17.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->fastai==2.1.5) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->fastai==2.1.5) (1.15.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai==2.1.5) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai==2.1.5) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->fastai==2.1.5) (0.10.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->fastai==2.1.5) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->fastai==2.1.5) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->fastai==2.1.5) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->fastai==2.1.5) (2.10)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (4.41.1)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (1.0.2)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (1.0.3)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (1.1.3)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (0.4.1)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (2.0.4)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (50.3.2)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (7.4.0)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (3.0.2)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (0.8.0)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy->fastai==2.1.5) (1.0.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->fastai==2.1.5) (2018.9)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy->fastai==2.1.5) (2.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy->fastai==2.1.5) (3.4.0)\n",
            "Installing collected packages: fastcore, fastai\n",
            "  Found existing installation: fastai 1.0.61\n",
            "    Uninstalling fastai-1.0.61:\n",
            "      Successfully uninstalled fastai-1.0.61\n",
            "Successfully installed fastai-2.1.5 fastcore-1.3.6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "fastai"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G1bRnxq2f2uY"
      },
      "source": [
        "from fastai.text.all import *"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "id": "QGl3Lj77hxjw",
        "outputId": "497ccbb4-f279-4ddd-f1be-cf2367ff90ca"
      },
      "source": [
        "path = untar_data(URLs.IMDB)\n",
        "path.ls()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(#7) [Path('/root/.fastai/data/imdb/train'),Path('/root/.fastai/data/imdb/tmp_lm'),Path('/root/.fastai/data/imdb/unsup'),Path('/root/.fastai/data/imdb/test'),Path('/root/.fastai/data/imdb/tmp_clas'),Path('/root/.fastai/data/imdb/README'),Path('/root/.fastai/data/imdb/imdb.vocab')]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "-S898vh-m3b-",
        "outputId": "03b6c1db-f510-4b8a-bba8-9eda73577b7a"
      },
      "source": [
        "dls_lm = TextDataLoaders.from_folder(path, is_lm=True, valid_pct=0.9)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "rofZF9dWnICU",
        "outputId": "208dce3c-9fc0-4059-8ecc-b603e511f3e7"
      },
      "source": [
        "learn = language_model_learner(dls_lm, AWD_LSTM, metrics=[accuracy, Perplexity()], path=path, wd=0.1).to_fp16()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "69ulMYFuoZxo",
        "outputId": "323280f0-289c-4856-d1a6-617dd3678abd"
      },
      "source": [
        "learn.fit_one_cycle(1, 1e-2)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>perplexity</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>4.618097</td>\n",
              "      <td>4.192440</td>\n",
              "      <td>0.279374</td>\n",
              "      <td>66.184052</td>\n",
              "      <td>15:56</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4M_KAPw9sU-M",
        "outputId": "be6f1a53-1641-4fd4-d385-5f4d61881b44"
      },
      "source": [
        "learn.save('1epoch')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Path('/root/.fastai/data/imdb/models/1epoch.pth')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_JkLLCSEohZK"
      },
      "source": [
        "learn = learn.load('1epoch')"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oZe64BmBsQul"
      },
      "source": [
        "learn = learn.load('1epoch')"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "vdNQg-3BsX2-",
        "outputId": "d7a6cb8c-fc42-4254-dae6-9e932d45ffea"
      },
      "source": [
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(10, 1e-3)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>perplexity</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>4.402622</td>\n",
              "      <td>4.124851</td>\n",
              "      <td>0.286855</td>\n",
              "      <td>61.858604</td>\n",
              "      <td>16:08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>4.328585</td>\n",
              "      <td>4.084512</td>\n",
              "      <td>0.291026</td>\n",
              "      <td>59.412922</td>\n",
              "      <td>16:10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>4.267871</td>\n",
              "      <td>4.052994</td>\n",
              "      <td>0.294537</td>\n",
              "      <td>57.569550</td>\n",
              "      <td>16:06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>4.185748</td>\n",
              "      <td>4.023940</td>\n",
              "      <td>0.298423</td>\n",
              "      <td>55.921005</td>\n",
              "      <td>16:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>4.120587</td>\n",
              "      <td>4.007434</td>\n",
              "      <td>0.300741</td>\n",
              "      <td>55.005566</td>\n",
              "      <td>16:01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>4.056413</td>\n",
              "      <td>3.991982</td>\n",
              "      <td>0.302514</td>\n",
              "      <td>54.162117</td>\n",
              "      <td>16:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>4.002944</td>\n",
              "      <td>3.981128</td>\n",
              "      <td>0.304236</td>\n",
              "      <td>53.577423</td>\n",
              "      <td>16:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>3.957130</td>\n",
              "      <td>3.973809</td>\n",
              "      <td>0.305204</td>\n",
              "      <td>53.186722</td>\n",
              "      <td>16:04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>3.917429</td>\n",
              "      <td>3.969718</td>\n",
              "      <td>0.305573</td>\n",
              "      <td>52.969616</td>\n",
              "      <td>16:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>3.896875</td>\n",
              "      <td>3.969859</td>\n",
              "      <td>0.305650</td>\n",
              "      <td>52.977066</td>\n",
              "      <td>16:05</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xmk63F5dscwf"
      },
      "source": [
        "learn.save_encoder('finetuned')"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "2ulx5cHURXEY",
        "outputId": "81b4c2d6-89b8-48f6-f5ad-d5a562fff2e9"
      },
      "source": [
        "TEXT = \"I liked this movie because\"\n",
        "N_WORDS = 40\n",
        "N_SENTENCES = 2\n",
        "preds = [learn.predict(TEXT, N_WORDS, temperature=0.75) \n",
        "         for _ in range(N_SENTENCES)]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uMdhVWaATEcN",
        "outputId": "461fb254-5e77-45fa-d4da-88c53539d4e6"
      },
      "source": [
        "print(\"\\n\".join(preds))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "i liked this movie because when i saw it i was still looking forward to watching it . The movie has some good moments and some really creepy moments but it 's far too exciting . The plot is so weak and boring\n",
            "i liked this movie because it was very good . i loved it . It was amazing . It was , of course , one of my favorite movies of all time . At one point i was shocked when Sam\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-EaIyf2PTHv2"
      },
      "source": [
        "dls_clas = TextDataLoaders.from_folder(untar_data(URLs.IMDB), valid='test', text_vocab=dls_lm.vocab)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ny8K2YRYTqBW"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H6mbzLwGTq0H"
      },
      "source": [
        "dls = TextDataLoaders.from_folder(untar_data(URLs.IMDB), valid='test')"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_C1TarJTZwo"
      },
      "source": [
        "learn = text_classifier_learner(dls, AWD_LSTM, drop_mult=0.5, metrics=accuracy)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ueBOj4ofTee8"
      },
      "source": [
        "learn = learn.load_encoder('finetuned')"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "ZXM6aNDATyD6",
        "outputId": "24e31e06-3d2d-43e8-c6c5-769b6cabb5c3"
      },
      "source": [
        "learn.fit_one_cycle(1, 2e-2)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.483253</td>\n",
              "      <td>0.257572</td>\n",
              "      <td>0.893800</td>\n",
              "      <td>04:18</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "qJDS7NYiTzg3",
        "outputId": "9d429d91-0095-4cad-ff57-fda1243f7d3d"
      },
      "source": [
        "learn.freeze_to(-2)\n",
        "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.278638</td>\n",
              "      <td>0.206865</td>\n",
              "      <td>0.919400</td>\n",
              "      <td>04:50</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "TyXC4SFvVOO8",
        "outputId": "a78c7e2e-2ea7-46a7-fdbb-6ba817949e2d"
      },
      "source": [
        "learn.freeze_to(-3)\n",
        "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.221458</td>\n",
              "      <td>0.175317</td>\n",
              "      <td>0.932800</td>\n",
              "      <td>06:45</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        },
        "id": "tzZ-kWKYWXNA",
        "outputId": "6a9d290f-5768-4401-80a3-e4d47d9bba52"
      },
      "source": [
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.178724</td>\n",
              "      <td>0.172487</td>\n",
              "      <td>0.934840</td>\n",
              "      <td>08:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.151952</td>\n",
              "      <td>0.170765</td>\n",
              "      <td>0.935920</td>\n",
              "      <td>08:17</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "e_tIEHd0Z-wu",
        "outputId": "c8af38df-1621-403e-822d-1f2a2104d81d"
      },
      "source": [
        " learn.predict(\"It is so bad!\")"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('neg', TensorText(0), TensorText([0.9874, 0.0126]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lXbB4ENqgbiK"
      },
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv('./total_data.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "_zWChDtbhwCc",
        "outputId": "939f4d0f-08bd-4023-c467-32ca70d1977d"
      },
      "source": [
        "dls = TextDataLoaders.from_csv(path='./', csv_fname='total_data.csv', text_col='content', label_col='class_label')"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u3gIMbCIg7C5"
      },
      "source": [
        "learn = text_classifier_learner(dls, AWD_LSTM, drop_mult=0.5, metrics=accuracy)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "jzcIpTunhYp2",
        "outputId": "a6ddf189-c433-4a17-8a8c-e36b9f22a92b"
      },
      "source": [
        "learn.fit_one_cycle(1, 1e-2)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.814359</td>\n",
              "      <td>1.225543</td>\n",
              "      <td>0.573935</td>\n",
              "      <td>00:14</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        },
        "id": "mgQxOpPKjT-G",
        "outputId": "d1658ed8-7a6b-42f5-fa7d-b01dffd55319"
      },
      "source": [
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(10, 1e-3)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.382857</td>\n",
              "      <td>1.060806</td>\n",
              "      <td>0.619549</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.052598</td>\n",
              "      <td>0.736687</td>\n",
              "      <td>0.746366</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.725875</td>\n",
              "      <td>0.670989</td>\n",
              "      <td>0.741855</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.772990</td>\n",
              "      <td>0.582112</td>\n",
              "      <td>0.787469</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.595475</td>\n",
              "      <td>0.611961</td>\n",
              "      <td>0.759399</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.602917</td>\n",
              "      <td>0.537157</td>\n",
              "      <td>0.784962</td>\n",
              "      <td>00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.610688</td>\n",
              "      <td>0.514513</td>\n",
              "      <td>0.804511</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.551586</td>\n",
              "      <td>0.509075</td>\n",
              "      <td>0.823559</td>\n",
              "      <td>00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.478207</td>\n",
              "      <td>0.510830</td>\n",
              "      <td>0.807519</td>\n",
              "      <td>00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.427497</td>\n",
              "      <td>0.521992</td>\n",
              "      <td>0.806015</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "id": "QAOl09j-liRX",
        "outputId": "71b955b7-dc02-4ef9-f760-12100b8d46fe"
      },
      "source": [
        "learn.fit_one_cycle(10, 1e-3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='5' class='' max='10' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      50.00% [5/10 01:43<01:43]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.969249</td>\n",
              "      <td>1.247478</td>\n",
              "      <td>0.671178</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.940988</td>\n",
              "      <td>1.057921</td>\n",
              "      <td>0.701253</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.945879</td>\n",
              "      <td>1.009664</td>\n",
              "      <td>0.707769</td>\n",
              "      <td>00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.882341</td>\n",
              "      <td>1.421873</td>\n",
              "      <td>0.706266</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.915932</td>\n",
              "      <td>1.112908</td>\n",
              "      <td>0.697243</td>\n",
              "      <td>00:22</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='102' class='' max='124' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      82.26% [102/124 00:14<00:03 0.8882]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpxnwE-qjl4L"
      },
      "source": [
        "learn.save_encoder('finetuned')"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "rJUKfFmLkZ_V",
        "outputId": "24e3fa02-a733-4943-f816-8ba15431601e"
      },
      "source": [
        "learn.fit_one_cycle(1, 2e-2)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.456029</td>\n",
              "      <td>1.495504</td>\n",
              "      <td>0.606516</td>\n",
              "      <td>00:19</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "xGm4FzW2kbaN",
        "outputId": "eef92605-4365-42ef-9c6a-157b3a196b99"
      },
      "source": [
        "learn.freeze_to(-2)\n",
        "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2))\n"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.165102</td>\n",
              "      <td>1.171023</td>\n",
              "      <td>0.657644</td>\n",
              "      <td>00:15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "id": "O1_WdL_qkiBl",
        "outputId": "e9401088-c3ca-4c77-bb89-c0df0c8b94e8"
      },
      "source": [
        "learn.freeze_to(-3)\n",
        "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))\n"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.995579</td>\n",
              "      <td>1.219023</td>\n",
              "      <td>0.676692</td>\n",
              "      <td>00:17</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        },
        "id": "tMjPZRwBkolc",
        "outputId": "764aa88a-f0e2-40f2-f5a5-b7e7861ecd43"
      },
      "source": [
        "learn.unfreeze()\n",
        "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>1.053785</td>\n",
              "      <td>1.066315</td>\n",
              "      <td>0.681203</td>\n",
              "      <td>00:19</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.015238</td>\n",
              "      <td>1.381374</td>\n",
              "      <td>0.637594</td>\n",
              "      <td>00:20</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xyT4j2W4kvUo"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}