Textclassification_with_XLNET.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import math\n",
    "import numpy as np\n",
    "from sklearn.metrics import classification_report\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "from tqdm import tqdm,trange\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 07:19:02.449230 140199490098944 file_utils.py:32] TensorFlow version 2.0.0-rc1 available.\n",
      "I0107 07:19:02.450446 140199490098944 file_utils.py:39] PyTorch version 1.1.0 available.\n"
     ]
    }
   ],
   "source": [
    "from transformers import (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras                2.3.1                 \n",
      "Keras-Applications   1.0.8                 \n",
      "Keras-Preprocessing  1.0.9                 \n",
      "torch                1.1.0                 \n",
      "transformers         2.2.0                 \n"
     ]
    }
   ],
   "source": [
    "# Check library version\n",
    "!pip list | grep -E 'transformers|torch|Keras'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook work with env:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Keras                2.3.1                 \n",
    "- torch                1.1.0                 \n",
    "- transformers         2.2.0      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, will introduce how to do text classification with XLNet, including:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Load and preprocess data\n",
    "- Parser data\n",
    "- Make training data\n",
    "- Train model\n",
    "- Evaluate result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tips:\n",
    " - Update PyTorch to 1.1.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Also this notebook come with a post [Text Classification with XLNet in Action](https://medium.com/@yingbiao/text-classification-with-xlnet-in-action-869029246f7e)**<br>\n",
    "**Feel free to check it, hope that it could help you.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load CSV data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"data/\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_file_address = \"data/text_classification_dataset.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.read_csv(data_file_address,sep=\",\",encoding=\"utf-8\",names=['labels','texts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['labels', 'texts'], dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>labels</th>\n",
       "      <th>texts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>god is great , the movie's not .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>. . . the whole thing succeeded only in making...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>light the candles , bring out the cake and don...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>the story may not be new , but australian dire...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>you live the mood rather than savour the story .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>. . . \" bowling for columbine \" remains a disq...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>occasionally amateurishly made but a winsome c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>by the time you reach the finale , you're like...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>the best way to hope for any chance of enjoyin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>something must have been lost in the translati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>maelstrom is strange and compelling , engrossi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0</td>\n",
       "      <td>for all the complications , it's all surprisin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>a pure participatory event that malnourished i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>examines its explosive subject matter as nonju...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>with we were soldiers , hollywood makes a vali...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>the movie's gloomy atmosphere is fascinating ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0</td>\n",
       "      <td>the movie's biggest shocks come from seeing fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0</td>\n",
       "      <td>the only thing scary about feardotcom is that ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>shamelessly resorting to pee-related sight gag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>' . . . both hokey and super-cool , and defini...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    labels                                              texts\n",
       "0        0                   god is great , the movie's not .\n",
       "1        0  . . . the whole thing succeeded only in making...\n",
       "2        1  light the candles , bring out the cake and don...\n",
       "3        1  the story may not be new , but australian dire...\n",
       "4        1   you live the mood rather than savour the story .\n",
       "5        1  . . . \" bowling for columbine \" remains a disq...\n",
       "6        1  occasionally amateurishly made but a winsome c...\n",
       "7        0  by the time you reach the finale , you're like...\n",
       "8        0  the best way to hope for any chance of enjoyin...\n",
       "9        0  something must have been lost in the translati...\n",
       "10       1  maelstrom is strange and compelling , engrossi...\n",
       "11       0  for all the complications , it's all surprisin...\n",
       "12       1  a pure participatory event that malnourished i...\n",
       "13       1  examines its explosive subject matter as nonju...\n",
       "14       1  with we were soldiers , hollywood makes a vali...\n",
       "15       0  the movie's gloomy atmosphere is fascinating ,...\n",
       "16       0  the movie's biggest shocks come from seeing fo...\n",
       "17       0  the only thing scary about feardotcom is that ...\n",
       "18       1  shamelessly resorting to pee-related sight gag...\n",
       "19       1  ' . . . both hokey and super-cool , and defini..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.head(n=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Have a look labels**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.labels.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    5331\n",
       "0    5331\n",
       "Name: labels, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Analyse the labels distribution\n",
    "df_data.labels.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parser data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Parser data into document structure**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"god is great , the movie's not .\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get sentence data\n",
    "sentences = df_data.texts.to_list()\n",
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "# Get tag labels data\n",
    "labels = df_data.labels.to_list()\n",
    "print(labels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make TAG name into index for training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a dict for mapping id to tag name\n",
    "#tag2idx = {t: i for i, t in enumerate(tags_vals)}\n",
    "\n",
    "# Recommend to set it by manual define, good for reusing\n",
    "# 0:negative, 1: positive\n",
    "tag2idx={'0': 0,\n",
    " '1': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'0': 0, '1': 1}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mapping index to name\n",
    "tag2name={tag2idx[key] : key for key in tag2idx.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: '0', 1: '1'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag2name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make tranning data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make raw data into trainable data for XLNet, including:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Set gpu environment\n",
    "- Load tokenizer and tokenize\n",
    "- Set 3 embedding, token embedding, mask word embedding, segmentation embedding\n",
    "- Split data set into train and validate, then send them to dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Set up gpu environment**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_gpu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remember to install sentencepiece with  'pip install sentencepiece'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manual define vocabulary address, if you download the model in local\n",
    "# The vocabulary can download from \"https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-spiece.model\"\n",
    "vocabulary = 'models/xlnet-base-cased/xlnet-base-cased-spiece.model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Len of the sentence must be the same as the training model\n",
    "# See model's 'max_position_embeddings' = 512\n",
    "max_len  = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# With cased model, set do_lower_case = False\n",
    "tokenizer = XLNetTokenizer(vocab_file=vocabulary,do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set text input embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- token id embedding\n",
    "- mask embedding\n",
    "- segment embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Embedding process was referred to [XLNet official repo](https://github.com/zihangdai/xlnet/blob/master/classifier_utils.py)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This process is huge differnent from BERT**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No.:0\n",
      "sentence: god is great , the movie's not .\n",
      "input_ids:[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7290, 27, 312, 17, 19, 18, 1432, 26, 23, 50, 17, 9, 4, 3, 4, 3]\n",
      "attention_masks:[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "segment_ids:[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]\n",
      "\n",
      "\n",
      "No.:1\n",
      "sentence: . . . the whole thing succeeded only in making me groggy .\n",
      "input_ids:[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 9, 17, 9, 17, 9, 18, 856, 554, 5741, 114, 25, 441, 110, 17, 7059, 13006, 17, 9, 4, 3, 4, 3]\n",
      "attention_masks:[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "segment_ids:[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]\n",
      "\n",
      "\n",
      "No.:2\n",
      "sentence: light the candles , bring out the cake and don't fret about the calories because there's precious little substance in birthday girl -- it's simply , and surprisingly , a nice , light treat .\n",
      "input_ids:[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 697, 18, 16241, 17, 19, 962, 78, 18, 6540, 21, 220, 26, 46, 22953, 75, 18, 10799, 149, 105, 26, 23, 8604, 293, 6997, 25, 4903, 1615, 17, 13, 13, 36, 26, 23, 1172, 17, 19, 21, 10653, 17, 19, 24, 2101, 17, 19, 697, 3935, 17, 9, 4, 3, 4, 3]\n",
      "attention_masks:[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "segment_ids:[4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "max_len  = 64\n",
    "\n",
    "full_input_ids = []\n",
    "full_input_masks = []\n",
    "full_segment_ids = []\n",
    "\n",
    "SEG_ID_A   = 0\n",
    "SEG_ID_B   = 1\n",
    "SEG_ID_CLS = 2\n",
    "SEG_ID_SEP = 3\n",
    "SEG_ID_PAD = 4\n",
    "\n",
    "UNK_ID = tokenizer.encode(\"<unk>\")[0]\n",
    "CLS_ID = tokenizer.encode(\"<cls>\")[0]\n",
    "SEP_ID = tokenizer.encode(\"<sep>\")[0]\n",
    "MASK_ID = tokenizer.encode(\"<mask>\")[0]\n",
    "EOD_ID = tokenizer.encode(\"<eod>\")[0]\n",
    "\n",
    "for i,sentence in enumerate(sentences):\n",
    "    # Tokenize sentence to token id list\n",
    "    tokens_a = tokenizer.encode(sentence)\n",
    "    \n",
    "    # Trim the len of text\n",
    "    if(len(tokens_a)>max_len-2):\n",
    "        tokens_a = tokens_a[:max_len-2]\n",
    "        \n",
    "        \n",
    "    tokens = []\n",
    "    segment_ids = []\n",
    "    \n",
    "    for token in tokens_a:\n",
    "        tokens.append(token)\n",
    "        segment_ids.append(SEG_ID_A)\n",
    "        \n",
    "    # Add <sep> token \n",
    "    tokens.append(SEP_ID)\n",
    "    segment_ids.append(SEG_ID_A)\n",
    "    \n",
    "    \n",
    "    # Add <cls> token\n",
    "    tokens.append(CLS_ID)\n",
    "    segment_ids.append(SEG_ID_CLS)\n",
    "    \n",
    "    input_ids = tokens\n",
    "    \n",
    "    # The mask has 0 for real tokens and 1 for padding tokens. Only real\n",
    "    # tokens are attended to.\n",
    "    input_mask = [0] * len(input_ids)\n",
    "\n",
    "    # Zero-pad up to the sequence length at fornt\n",
    "    if len(input_ids) < max_len:\n",
    "        delta_len = max_len - len(input_ids)\n",
    "        input_ids = [0] * delta_len + input_ids\n",
    "        input_mask = [1] * delta_len + input_mask\n",
    "        segment_ids = [SEG_ID_PAD] * delta_len + segment_ids\n",
    "\n",
    "    assert len(input_ids) == max_len\n",
    "    assert len(input_mask) == max_len\n",
    "    assert len(segment_ids) == max_len\n",
    "    \n",
    "    full_input_ids.append(input_ids)\n",
    "    full_input_masks.append(input_mask)\n",
    "    full_segment_ids.append(segment_ids)\n",
    "    \n",
    "    if 3 > i:\n",
    "        print(\"No.:%d\"%(i))\n",
    "        print(\"sentence: %s\"%(sentence))\n",
    "        print(\"input_ids:%s\"%(input_ids))\n",
    "        print(\"attention_masks:%s\"%(input_mask))\n",
    "        print(\"segment_ids:%s\"%(segment_ids))\n",
    "        print(\"\\n\")\n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set label embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "# Make label into id\n",
    "tags = [tag2idx[str(lab)] for lab in labels]\n",
    "print(tags[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data into train and validate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "70% for training, 30% for validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Split all data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs, val_inputs, tr_tags, val_tags,tr_masks, val_masks,tr_segs, val_segs = train_test_split(full_input_ids, tags,full_input_masks,full_segment_ids, \n",
    "                                                            random_state=4, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7463, 3199, 7463, 3199)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tr_inputs),len(val_inputs),len(tr_segs),len(val_segs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Set data into tensor**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not recommend tensor.to(device) at this process, since it will run out of GPU memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(tr_inputs)\n",
    "val_inputs = torch.tensor(val_inputs)\n",
    "tr_tags = torch.tensor(tr_tags)\n",
    "val_tags = torch.tensor(val_tags)\n",
    "tr_masks = torch.tensor(tr_masks)\n",
    "val_masks = torch.tensor(val_masks)\n",
    "tr_segs = torch.tensor(tr_segs)\n",
    "val_segs = torch.tensor(val_segs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Put data into data loader**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set batch num\n",
    "batch_num = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set token embedding, attention embedding, segment embedding\n",
    "train_data = TensorDataset(tr_inputs, tr_masks,tr_segs, tr_tags)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "# Drop last can make batch training better for the last one\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_num,drop_last=True)\n",
    "\n",
    "valid_data = TensorDataset(val_inputs, val_masks,val_segs, val_tags)\n",
    "valid_sampler = SequentialSampler(valid_data)\n",
    "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=batch_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load XLNet model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In this document, contain confg(txt) and weight(bin) files\n",
    "# The folder must contain: pytorch_model.bin, config.json\n",
    "model_file_address = 'models/xlnet-base-cased'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 07:19:14.816964 140199490098944 configuration_utils.py:149] loading configuration file models/xlnet-base-cased/config.json\n",
      "I0107 07:19:14.819163 140199490098944 configuration_utils.py:169] Model config {\n",
      "  \"attn_type\": \"bi\",\n",
      "  \"bi_data\": false,\n",
      "  \"clamp_len\": -1,\n",
      "  \"d_head\": 64,\n",
      "  \"d_inner\": 3072,\n",
      "  \"d_model\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"end_n_top\": 5,\n",
      "  \"ff_activation\": \"gelu\",\n",
      "  \"finetuning_task\": null,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"is_decoder\": false,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"mem_len\": null,\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"n_token\": 32000,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"reuse_len\": null,\n",
      "  \"same_length\": false,\n",
      "  \"start_n_top\": 5,\n",
      "  \"summary_activation\": \"tanh\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"last\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"torchscript\": false,\n",
      "  \"untie_r\": true,\n",
      "  \"use_bfloat16\": false\n",
      "}\n",
      "\n",
      "I0107 07:19:14.821584 140199490098944 modeling_utils.py:384] loading weights file models/xlnet-base-cased/pytorch_model.bin\n",
      "I0107 07:19:18.394675 140199490098944 modeling_utils.py:457] Weights of XLNetForSequenceClassification not initialized from pretrained model: ['sequence_summary.summary.weight', 'sequence_summary.summary.bias', 'logits_proj.weight', 'logits_proj.bias']\n",
      "I0107 07:19:18.395839 140199490098944 modeling_utils.py:460] Weights from pretrained model not used in XLNetForSequenceClassification: ['lm_loss.weight', 'lm_loss.bias']\n"
     ]
    }
   ],
   "source": [
    "# Will load config and weight with from_pretrained()\n",
    "# Recommand download the model before using\n",
    "# Download model from \"https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-pytorch_model.bin\"\n",
    "# Download model from \"https://s3.amazonaws.com/models.huggingface.co/bert/xlnet-base-cased-config.json\" \n",
    "# ATTENTION!, rename \"xlnet-base-cased-pytorch_model.bin\" into \"pytorch_model.bin\"\n",
    "# rename \"xlnet-base-cased-config.json\" inot \"config.json\"\n",
    "model = XLNetForSequenceClassification.from_pretrained(model_file_address,num_labels=len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "model;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set model to GPU,if you are using GPU machine\n",
    "model.to(device);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add multi GPU support\n",
    "if n_gpu >1:\n",
    "    model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set epoch and grad max num\n",
    "epochs = 5\n",
    "max_grad_norm = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cacluate train optimiazaion num\n",
    "num_train_optimization_steps = int( math.ceil(len(tr_inputs) / batch_num) / 1) * epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set fine tuning method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Manual optimizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True: fine tuning all the layers \n",
    "# False: only fine tuning the classifier layers\n",
    "# Since XLNet in 'pytorch_transformer' did not contian classifier layers\n",
    "# FULL_FINETUNING = True need to set True\n",
    "FULL_FINETUNING = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "if FULL_FINETUNING:\n",
    "    # Fine tune model all layer parameters\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    # Only fine tune classifier parameters\n",
    "    param_optimizer = list(model.classifier.named_parameters()) \n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-tuing model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TRAIN loop\n",
    "model.train();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 7463\n",
      "  Batch size = 32\n",
      "  Num steps = 1170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:  20%|██        | 1/5 [00:46<03:05, 46.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.4623915070244171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:  40%|████      | 2/5 [01:33<02:19, 46.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.27201624612759623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:  60%|██████    | 3/5 [02:19<01:33, 46.51s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.18195790735117356\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:  80%|████████  | 4/5 [03:05<00:46, 46.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.12496821583116771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 100%|██████████| 5/5 [03:52<00:00, 46.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.08649337041899242\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"***** Running training *****\")\n",
    "print(\"  Num examples = %d\"%(len(tr_inputs)))\n",
    "print(\"  Batch size = %d\"%(batch_num))\n",
    "print(\"  Num steps = %d\"%(num_train_optimization_steps))\n",
    "for _ in trange(epochs,desc=\"Epoch\"):\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_segs,b_labels = batch\n",
    "        \n",
    "        # forward pass\n",
    "        outputs = model(input_ids =b_input_ids,token_type_ids=b_segs, input_mask = b_input_mask,labels=b_labels)\n",
    "        loss, logits = outputs[:2]\n",
    "        if n_gpu>1:\n",
    "            # When multi gpu, average it\n",
    "            loss = loss.mean()\n",
    "        \n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        \n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        \n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        \n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "xlnet_out_address = 'models/xlnet_out_model/tc02'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make dir if not exits\n",
    "if not os.path.exists(xlnet_out_address):\n",
    "        os.makedirs(xlnet_out_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model, configuration and tokenizer\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If we save using the predefined names, we can load using `from_pretrained`\n",
    "output_model_file = os.path.join(xlnet_out_address, \"pytorch_model.bin\")\n",
    "output_config_file = os.path.join(xlnet_out_address, \"config.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('models/xlnet_out_model/tc02/spiece.model',)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save model into file\n",
    "torch.save(model_to_save.state_dict(), output_model_file)\n",
    "model_to_save.config.to_json_file(output_config_file)\n",
    "tokenizer.save_vocabulary(xlnet_out_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 07:23:15.511597 140199490098944 configuration_utils.py:149] loading configuration file models/xlnet_out_model/tc02/config.json\n",
      "I0107 07:23:15.513851 140199490098944 configuration_utils.py:169] Model config {\n",
      "  \"attn_type\": \"bi\",\n",
      "  \"bi_data\": false,\n",
      "  \"clamp_len\": -1,\n",
      "  \"d_head\": 64,\n",
      "  \"d_inner\": 3072,\n",
      "  \"d_model\": 768,\n",
      "  \"dropout\": 0.1,\n",
      "  \"end_n_top\": 5,\n",
      "  \"ff_activation\": \"gelu\",\n",
      "  \"finetuning_task\": null,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"is_decoder\": false,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"mem_len\": null,\n",
      "  \"n_head\": 12,\n",
      "  \"n_layer\": 12,\n",
      "  \"n_token\": 32000,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"reuse_len\": null,\n",
      "  \"same_length\": false,\n",
      "  \"start_n_top\": 5,\n",
      "  \"summary_activation\": \"tanh\",\n",
      "  \"summary_last_dropout\": 0.1,\n",
      "  \"summary_type\": \"last\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"torchscript\": false,\n",
      "  \"untie_r\": true,\n",
      "  \"use_bfloat16\": false\n",
      "}\n",
      "\n",
      "I0107 07:23:15.515127 140199490098944 modeling_utils.py:384] loading weights file models/xlnet_out_model/tc02/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "model = XLNetForSequenceClassification.from_pretrained(xlnet_out_address,num_labels=len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set model to GPU\n",
    "model.to(device);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "if n_gpu >1:\n",
    "    model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eval model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evalue loop\n",
    "model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set acc funtion\n",
    "def accuracy(out, labels):\n",
    "    outputs = np.argmax(out, axis=1)\n",
    "    return np.sum(outputs == labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running evaluation *****\n",
      "  Num examples =3199\n",
      "  Batch size = 32\n",
      "***** Eval results *****\n",
      "  eval_accuracy = 0.8783994998437011\n",
      "  eval_loss = 0.5584006253629923\n",
      "  loss = 0.08649337041899242\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.86      0.87      1586\n",
      "           1       0.86      0.90      0.88      1613\n",
      "\n",
      "    accuracy                           0.88      3199\n",
      "   macro avg       0.88      0.88      0.88      3199\n",
      "weighted avg       0.88      0.88      0.88      3199\n",
      "\n"
     ]
    }
   ],
   "source": [
    "eval_loss, eval_accuracy = 0, 0\n",
    "nb_eval_steps, nb_eval_examples = 0, 0\n",
    "\n",
    "y_true = []\n",
    "y_predict = []\n",
    "print(\"***** Running evaluation *****\")\n",
    "print(\"  Num examples ={}\".format(len(val_inputs)))\n",
    "print(\"  Batch size = {}\".format(batch_num))\n",
    "for step, batch in enumerate(valid_dataloader):\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    b_input_ids, b_input_mask, b_segs,b_labels = batch\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        outputs = model(input_ids =b_input_ids,token_type_ids=b_segs, input_mask = b_input_mask,labels=b_labels)\n",
    "        tmp_eval_loss, logits = outputs[:2]\n",
    "    \n",
    "    # Get textclassification predict result\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    label_ids = b_labels.to('cpu').numpy()\n",
    "    tmp_eval_accuracy = accuracy(logits, label_ids)\n",
    "#     print(tmp_eval_accuracy)\n",
    "#     print(np.argmax(logits, axis=1))\n",
    "#     print(label_ids)\n",
    "    \n",
    "    # Save predict and real label reuslt for analyze\n",
    "    for predict in np.argmax(logits, axis=1):\n",
    "        y_predict.append(predict)\n",
    "        \n",
    "    for real_result in label_ids.tolist():\n",
    "        y_true.append(real_result)\n",
    "\n",
    "    \n",
    "    eval_loss += tmp_eval_loss.mean().item()\n",
    "    eval_accuracy += tmp_eval_accuracy\n",
    "   \n",
    "    nb_eval_steps += 1\n",
    "    \n",
    "    \n",
    "eval_loss = eval_loss / nb_eval_steps\n",
    "eval_accuracy = eval_accuracy / len(val_inputs)\n",
    "loss = tr_loss/nb_tr_steps \n",
    "result = {'eval_loss': eval_loss,\n",
    "                  'eval_accuracy': eval_accuracy,\n",
    "                  'loss': loss}\n",
    "report = classification_report(y_pred=np.array(y_predict),y_true=np.array(y_true))\n",
    "\n",
    "# Save the report into file\n",
    "output_eval_file = os.path.join(xlnet_out_address, \"eval_results.txt\")\n",
    "with open(output_eval_file, \"w\") as writer:\n",
    "    print(\"***** Eval results *****\")\n",
    "    for key in sorted(result.keys()):\n",
    "        print(\"  %s = %s\"%(key, str(result[key])))\n",
    "        writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
    "        \n",
    "    print(report)\n",
    "    writer.write(\"\\n\\n\")  \n",
    "    writer.write(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}