NER_with_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import math\n",
    "import numpy as np\n",
    "from seqeval.metrics import f1_score\n",
    "from seqeval.metrics import classification_report,accuracy_score,f1_score\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "I0507 16:24:41.160771 139888124225280 file_utils.py:41] PyTorch version 1.1.0 available.\n",
      "I0507 16:24:41.162539 139888124225280 file_utils.py:57] TensorFlow version 2.0.0 available.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import os\n",
    "from tqdm import tqdm,trange\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from sklearn.model_selection import train_test_split\n",
    "from transformers import BertTokenizer, BertConfig\n",
    "from transformers import BertForTokenClassification, AdamW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras                2.3.1                 \n",
      "Keras-Applications   1.0.8                 \n",
      "Keras-Preprocessing  1.1.0                 \n",
      "torch                1.1.0                 \n",
      "transformers         2.5.1                 \n",
      "\u001b[33mWARNING: You are using pip version 19.3.1; however, version 20.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Check library version\n",
    "!pip list | grep -E 'transformers|torch|Keras'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook work with env:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Keras                2.3.1                 \n",
    "- torch                1.1.0                 \n",
    "- transformers         2.5.1      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, will introduce how to do NER with BERT, including:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Load and preprocess data\n",
    "- Parser data\n",
    "- Make training data\n",
    "- Train model\n",
    "- Evaluate result\n",
    "- **Inference**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tips:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Update to transformer==2.5.1\n",
    "- When come across OOV,you will find that BERT word piece tokenize method can help a lot\n",
    "- Case model will be litter better than uncase model for English"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Also this notebook come with a post [NER with BERT in Action](https://medium.com/@yingbiao/ner-with-bert-in-action-936ff275bc73)**<br>\n",
    "**Feel free to check it, hope that it could help you.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load CSV data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"data/\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_file_address = \"data/ner_dataset.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fillna method can make same sentence with same sentence name\n",
    "df_data = pd.read_csv(data_file_address,sep=\",\",encoding=\"latin1\").fillna(method='ffill')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Sentence #', 'Word', 'POS', 'Tag'], dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence #</th>\n",
       "      <th>Word</th>\n",
       "      <th>POS</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>Thousands</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>demonstrators</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>have</td>\n",
       "      <td>VBP</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>marched</td>\n",
       "      <td>VBN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>through</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>London</td>\n",
       "      <td>NNP</td>\n",
       "      <td>B-geo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>to</td>\n",
       "      <td>TO</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>protest</td>\n",
       "      <td>VB</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>the</td>\n",
       "      <td>DT</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>war</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>in</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>Iraq</td>\n",
       "      <td>NNP</td>\n",
       "      <td>B-geo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>and</td>\n",
       "      <td>CC</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>demand</td>\n",
       "      <td>VB</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>the</td>\n",
       "      <td>DT</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>withdrawal</td>\n",
       "      <td>NN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>of</td>\n",
       "      <td>IN</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>British</td>\n",
       "      <td>JJ</td>\n",
       "      <td>B-gpe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Sentence: 1</td>\n",
       "      <td>troops</td>\n",
       "      <td>NNS</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Sentence #           Word  POS    Tag\n",
       "0   Sentence: 1      Thousands  NNS      O\n",
       "1   Sentence: 1             of   IN      O\n",
       "2   Sentence: 1  demonstrators  NNS      O\n",
       "3   Sentence: 1           have  VBP      O\n",
       "4   Sentence: 1        marched  VBN      O\n",
       "5   Sentence: 1        through   IN      O\n",
       "6   Sentence: 1         London  NNP  B-geo\n",
       "7   Sentence: 1             to   TO      O\n",
       "8   Sentence: 1        protest   VB      O\n",
       "9   Sentence: 1            the   DT      O\n",
       "10  Sentence: 1            war   NN      O\n",
       "11  Sentence: 1             in   IN      O\n",
       "12  Sentence: 1           Iraq  NNP  B-geo\n",
       "13  Sentence: 1            and   CC      O\n",
       "14  Sentence: 1         demand   VB      O\n",
       "15  Sentence: 1            the   DT      O\n",
       "16  Sentence: 1     withdrawal   NN      O\n",
       "17  Sentence: 1             of   IN      O\n",
       "18  Sentence: 1        British   JJ  B-gpe\n",
       "19  Sentence: 1         troops  NNS      O"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.head(n=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Have a look POS cat**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['NNS', 'IN', 'VBP', 'VBN', 'NNP', 'TO', 'VB', 'DT', 'NN', 'CC',\n",
       "       'JJ', '.', 'VBD', 'WP', '``', 'CD', 'PRP', 'VBZ', 'POS', 'VBG',\n",
       "       'RB', ',', 'WRB', 'PRP$', 'MD', 'WDT', 'JJR', ':', 'JJS', 'WP$',\n",
       "       'RP', 'PDT', 'NNPS', 'EX', 'RBS', 'LRB', 'RRB', '$', 'RBR', ';',\n",
       "       'UH', 'FW'], dtype=object)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.POS.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Have a look TAG cat**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['O', 'B-geo', 'B-gpe', 'B-per', 'I-geo', 'B-org', 'I-org', 'B-tim',\n",
       "       'B-art', 'I-art', 'I-per', 'I-gpe', 'I-tim', 'B-nat', 'B-eve',\n",
       "       'I-eve', 'I-nat'], dtype=object)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.Tag.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(47959, 35178, 42, 17)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Analyse summary of data\n",
    "df_data['Sentence #'].nunique(), df_data.Word.nunique(), df_data.POS.nunique(), df_data.Tag.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "O        887908\n",
       "B-geo     37644\n",
       "B-tim     20333\n",
       "B-org     20143\n",
       "I-per     17251\n",
       "B-per     16990\n",
       "I-org     16784\n",
       "B-gpe     15870\n",
       "I-geo      7414\n",
       "I-tim      6528\n",
       "B-art       402\n",
       "B-eve       308\n",
       "I-art       297\n",
       "I-eve       253\n",
       "B-nat       201\n",
       "I-gpe       198\n",
       "I-nat        51\n",
       "Name: Tag, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Analyse the Tag distribution\n",
    "df_data.Tag.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explain tag\n",
    "As show above, there are two parts for the tag name: \"position\"-\"meaning\"\n",
    "- B: begin, word at the first  position\n",
    "- I: middle, word not at the first position,especially for phase\n",
    "- time: time, meaning time\n",
    "- per: person, meaning people name\n",
    "- geo: geography, meaning location name\n",
    "- O: mean other, set as a default tag\n",
    "<br>......"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parser data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Parser data into document structure**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SentenceGetter(object):\n",
    "    \n",
    "    def __init__(self, data):\n",
    "        self.n_sent = 1\n",
    "        self.data = data\n",
    "        self.empty = False\n",
    "        agg_func = lambda s: [(w, p, t) for w, p, t in zip(s[\"Word\"].values.tolist(),\n",
    "                                                           s[\"POS\"].values.tolist(),\n",
    "                                                           s[\"Tag\"].values.tolist())]\n",
    "        self.grouped = self.data.groupby(\"Sentence #\").apply(agg_func)\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "    \n",
    "    def get_next(self):\n",
    "        try:\n",
    "            s = self.grouped[\"Sentence: {}\".format(self.n_sent)]\n",
    "            self.n_sent += 1\n",
    "            return s\n",
    "        except:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get full document data struce\n",
    "getter = SentenceGetter(df_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Thousands',\n",
       " 'of',\n",
       " 'demonstrators',\n",
       " 'have',\n",
       " 'marched',\n",
       " 'through',\n",
       " 'London',\n",
       " 'to',\n",
       " 'protest',\n",
       " 'the',\n",
       " 'war',\n",
       " 'in',\n",
       " 'Iraq',\n",
       " 'and',\n",
       " 'demand',\n",
       " 'the',\n",
       " 'withdrawal',\n",
       " 'of',\n",
       " 'British',\n",
       " 'troops',\n",
       " 'from',\n",
       " 'that',\n",
       " 'country',\n",
       " '.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get sentence data\n",
    "sentences = [[s[0] for s in sent] for sent in getter.sentences]\n",
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['NNS', 'IN', 'NNS', 'VBP', 'VBN', 'IN', 'NNP', 'TO', 'VB', 'DT', 'NN', 'IN', 'NNP', 'CC', 'VB', 'DT', 'NN', 'IN', 'JJ', 'NNS', 'IN', 'DT', 'NN', '.']\n"
     ]
    }
   ],
   "source": [
    "# Get pos data\n",
    "poses = [[s[1] for s in sent] for sent in getter.sentences]\n",
    "print(poses[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['O', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'O', 'O', 'O', 'B-geo', 'O', 'O', 'O', 'O', 'O', 'B-gpe', 'O', 'O', 'O', 'O', 'O']\n"
     ]
    }
   ],
   "source": [
    "# Get tag labels data\n",
    "labels = [[s[2] for s in sent] for sent in getter.sentences]\n",
    "print(labels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make TAG name into index for training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_vals = list(set(df_data[\"Tag\"].values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add X  label for word piece support\n",
    "# Add [CLS] and [SEP] as BERT need\n",
    "tags_vals.append('X')\n",
    "tags_vals.append('[CLS]')\n",
    "tags_vals.append('[SEP]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_vals = set(tags_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'B-art',\n",
       " 'B-eve',\n",
       " 'B-geo',\n",
       " 'B-gpe',\n",
       " 'B-nat',\n",
       " 'B-org',\n",
       " 'B-per',\n",
       " 'B-tim',\n",
       " 'I-art',\n",
       " 'I-eve',\n",
       " 'I-geo',\n",
       " 'I-gpe',\n",
       " 'I-nat',\n",
       " 'I-org',\n",
       " 'I-per',\n",
       " 'I-tim',\n",
       " 'O',\n",
       " 'X',\n",
       " '[CLS]',\n",
       " '[SEP]'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_vals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a dict for mapping id to tag name\n",
    "#tag2idx = {t: i for i, t in enumerate(tags_vals)}\n",
    "\n",
    "# Recommend to set it by manual define, good for reusing\n",
    "tag2idx={'B-art': 14,\n",
    " 'B-eve': 16,\n",
    " 'B-geo': 0,\n",
    " 'B-gpe': 13,\n",
    " 'B-nat': 12,\n",
    " 'B-org': 10,\n",
    " 'B-per': 4,\n",
    " 'B-tim': 2,\n",
    " 'I-art': 5,\n",
    " 'I-eve': 7,\n",
    " 'I-geo': 15,\n",
    " 'I-gpe': 8,\n",
    " 'I-nat': 11,\n",
    " 'I-org': 3,\n",
    " 'I-per': 6,\n",
    " 'I-tim': 1,\n",
    " 'X':17,\n",
    " 'O': 9,\n",
    " '[CLS]':18,\n",
    " '[SEP]':19}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'B-art': 14,\n",
       " 'B-eve': 16,\n",
       " 'B-geo': 0,\n",
       " 'B-gpe': 13,\n",
       " 'B-nat': 12,\n",
       " 'B-org': 10,\n",
       " 'B-per': 4,\n",
       " 'B-tim': 2,\n",
       " 'I-art': 5,\n",
       " 'I-eve': 7,\n",
       " 'I-geo': 15,\n",
       " 'I-gpe': 8,\n",
       " 'I-nat': 11,\n",
       " 'I-org': 3,\n",
       " 'I-per': 6,\n",
       " 'I-tim': 1,\n",
       " 'O': 9,\n",
       " 'X': 17,\n",
       " '[CLS]': 18,\n",
       " '[SEP]': 19}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mapping index to name\n",
    "tag2name={tag2idx[key] : key for key in tag2idx.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make training data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make raw data into trainable data for BERT, including:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Set gpu environment\n",
    "- Load tokenizer and tokenize\n",
    "- Set 3 embedding, token embedding, mask word embedding, segmentation embedding\n",
    "- Split data set into train and validate, then send them to dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Set up gpu environment**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_gpu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can download the tokenizer file into local folder first :\n",
    "- [vocab.txt](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Manual define vocabulary address, if you download the tokenzier file in local\n",
    "# vocab.txt, download from: https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt\n",
    "vocabulary = \"models/bert-base-cased/vocab.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Len of the sentence must be not bigger than the training model\n",
    "# See model's 'max_position_embeddings' = 512\n",
    "max_len  = 45"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load tokenizer, with manual file address or pretrained address\n",
    "tokenizer=BertTokenizer(vocab_file=vocabulary,do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Tokenizer text**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- In hunggieface for bert, when come across OOV, will word piece the word\n",
    "- We need to adjust the labels base on the tokenize result, “##abc” need to set label \"X\" \n",
    "- Need to set \"[CLS]\" at front and \"[SEP]\" at the end, as what the paper do, [BERT indexer should add [CLS] and [SEP] tokens](https://github.com/allenai/allennlp/issues/2141)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No.0,len:28\n",
      "texts:[CLS] Thousands of demons ##tra ##tors have marched through London to protest the war in Iraq and demand the withdrawal of British troops from that country . [SEP]\n",
      "No.0,len:28\n",
      "lables:[CLS] O O O X X O O O B-geo O O O O O B-geo O O O O O B-gpe O O O O O [SEP]\n",
      "No.1,len:29\n",
      "texts:[CLS] Iranian officials say they expect to get access to sealed sensitive parts of the plant Wednesday , after an I ##A ##EA surveillance system begins functioning . [SEP]\n",
      "No.1,len:29\n",
      "lables:[CLS] B-gpe O O O O O O O O O O O O O O B-tim O O O B-org X X O O O O O [SEP]\n",
      "No.2,len:44\n",
      "texts:[CLS] He ##lic ##op ##ter guns ##hips Saturday pounded militant hide ##outs in the Or ##ak ##zai tribal region , where many Taliban militants are believed to have fled to avoid an earlier military offensive in nearby South W ##azi ##rist ##an . [SEP]\n",
      "No.2,len:44\n",
      "lables:[CLS] O X X X O X B-tim O O O X O O B-geo X X O O O O O B-org O O O O O O O O O O O O O O B-geo I-geo X X X O [SEP]\n",
      "No.3,len:16\n",
      "texts:[CLS] They left after a tense hour - long stand ##off with riot police . [SEP]\n",
      "No.3,len:16\n",
      "lables:[CLS] O O O O O O X X O X O O O O [SEP]\n",
      "No.4,len:47\n",
      "texts:[CLS] U . N . relief coordinator Jan E ##gel ##and said Sunday , U . S . , Indonesian and Australian military helicopters are ferry ##ing out food and supplies to remote areas of western Ace ##h province that ground crews can not reach . [SEP]\n",
      "No.4,len:47\n",
      "lables:[CLS] B-geo X X X O O B-per I-per X X O B-tim O B-geo X X X O B-gpe O B-gpe O O O O X O O O O O O O O O B-geo X O O O O O O O O [SEP]\n"
     ]
    }
   ],
   "source": [
    "tokenized_texts = []\n",
    "word_piece_labels = []\n",
    "i_inc = 0\n",
    "for word_list,label in (zip(sentences,labels)):\n",
    "    temp_lable = []\n",
    "    temp_token = []\n",
    "    \n",
    "    # Add [CLS] at the front \n",
    "    temp_lable.append('[CLS]')\n",
    "    temp_token.append('[CLS]')\n",
    "    \n",
    "    for word,lab in zip(word_list,label):\n",
    "        token_list = tokenizer.tokenize(word)\n",
    "        for m,token in enumerate(token_list):\n",
    "            temp_token.append(token)\n",
    "            if m==0:\n",
    "                temp_lable.append(lab)\n",
    "            else:\n",
    "                temp_lable.append('X')  \n",
    "                \n",
    "    # Add [SEP] at the end\n",
    "    temp_lable.append('[SEP]')\n",
    "    temp_token.append('[SEP]')\n",
    "    \n",
    "    tokenized_texts.append(temp_token)\n",
    "    word_piece_labels.append(temp_lable)\n",
    "    \n",
    "    if 5 > i_inc:\n",
    "        print(\"No.%d,len:%d\"%(i_inc,len(temp_token)))\n",
    "        print(\"texts:%s\"%(\" \".join(temp_token)))\n",
    "        print(\"No.%d,len:%d\"%(i_inc,len(temp_lable)))\n",
    "        print(\"lables:%s\"%(\" \".join(temp_lable)))\n",
    "    i_inc +=1\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set token embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pad or trim the text and label to fit the need for max len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  101 26159  1104  8568  4487  5067  1138  9639  1194  1498  1106  5641\n",
      "  1103  1594  1107  5008  1105  4555  1103 10602  1104  1418  2830  1121\n",
      "  1115  1583   119   102     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0]\n"
     ]
    }
   ],
   "source": [
    "# Make text token into id\n",
    "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                          maxlen=max_len, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
    "print(input_ids[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18  9  9  9 17 17  9  9  9  0  9  9  9  9  9  0  9  9  9  9  9 13  9  9\n",
      "  9  9  9 19  9  9  9  9  9  9  9  9  9  9  9  9  9  9  9  9  9]\n"
     ]
    }
   ],
   "source": [
    "# Make label into id, pad with \"O\" meaning others\n",
    "tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in word_piece_labels],\n",
    "                     maxlen=max_len, value=tag2idx[\"O\"], padding=\"post\",\n",
    "                     dtype=\"long\", truncating=\"post\")\n",
    "print(tags[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set mask word embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For fine tune of predict, with token mask is 1,pad token is 0\n",
    "attention_masks = [[int(i>0) for i in ii] for ii in input_ids]\n",
    "attention_masks[0];"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set segment embedding(Seem like for sequance tagging task, it's not necessary to make this embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since only one sentence, all the segment set to 0\n",
    "segment_ids = [[0] * len(input_id) for input_id in input_ids]\n",
    "segment_ids[0];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data into train and validate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "70% for training, 30% for validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Split all data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs, val_inputs, tr_tags, val_tags,tr_masks, val_masks,tr_segs, val_segs = train_test_split(input_ids, tags,attention_masks,segment_ids, \n",
    "                                                            random_state=4, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33571, 14388, 33571, 14388)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tr_inputs),len(val_inputs),len(tr_segs),len(val_segs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Set data into tensor**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not recommend tensor.to(device) at this process, since it will run out of GPU memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(tr_inputs)\n",
    "val_inputs = torch.tensor(val_inputs)\n",
    "tr_tags = torch.tensor(tr_tags)\n",
    "val_tags = torch.tensor(val_tags)\n",
    "tr_masks = torch.tensor(tr_masks)\n",
    "val_masks = torch.tensor(val_masks)\n",
    "tr_segs = torch.tensor(tr_segs)\n",
    "val_segs = torch.tensor(val_segs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Put data into data loader**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set batch num\n",
    "batch_num = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only set token embedding, attention embedding, no segment embedding\n",
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "# Drop last can make batch training better for the last one\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_num,drop_last=True)\n",
    "\n",
    "valid_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "valid_sampler = SequentialSampler(valid_data)\n",
    "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=batch_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- You can download the model require files into local folder first\n",
    "- pytorch_model.bin: [pytorch_model.bin](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin)\n",
    "- config.json: [config.json](https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load BERT model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# In this folder, contain model confg(json) and model weight(bin) files\n",
    "# pytorch_model.bin, download from: https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin\n",
    "# config.json, downlaod from: https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json\n",
    "model_file_address = 'models/bert-base-cased'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 07:03:15.364966 139764591216384 configuration_utils.py:149] loading configuration file models/bert-base-cased/config.json\n",
      "I0107 07:03:15.367635 139764591216384 configuration_utils.py:169] Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 20,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0107 07:03:15.369343 139764591216384 modeling_utils.py:384] loading weights file models/bert-base-cased/pytorch_model.bin\n",
      "I0107 07:03:20.046349 139764591216384 modeling_utils.py:457] Weights of BertForTokenClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "I0107 07:03:20.047613 139764591216384 modeling_utils.py:460] Weights from pretrained model not used in BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    }
   ],
   "source": [
    "# Will load config and weight with from_pretrained()\n",
    "model = BertForTokenClassification.from_pretrained(model_file_address,num_labels=len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "model;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set model to GPU,if you are using GPU machine\n",
    "model.cuda();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add multi GPU support\n",
    "if n_gpu >1:\n",
    "    model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set epoch and grad max num\n",
    "epochs = 5\n",
    "max_grad_norm = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cacluate train optimiazaion num\n",
    "num_train_optimization_steps = int( math.ceil(len(tr_inputs) / batch_num) / 1) * epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set fine tuning method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Manual optimizer**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True: fine tuning all the layers \n",
    "# False: only fine tuning the classifier layers\n",
    "FULL_FINETUNING = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "if FULL_FINETUNING:\n",
    "    # Fine tune model all layer parameters\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    # Only fine tune classifier parameters\n",
    "    param_optimizer = list(model.classifier.named_parameters()) \n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fine-tuning model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TRAIN loop\n",
    "model.train();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:   0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 33571\n",
      "  Batch size = 32\n",
      "  Num steps = 5250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  20%|██        | 1/5 [02:11<08:47, 131.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.14100488066829536\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  40%|████      | 2/5 [04:23<06:35, 131.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.06855936513217548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  60%|██████    | 3/5 [06:35<04:23, 131.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.05256270733817318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "Epoch:  80%|████████  | 4/5 [08:46<02:11, 131.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.0404466844272149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch: 100%|██████████| 5/5 [10:56<00:00, 131.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train loss: 0.030035572352457802\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"***** Running training *****\")\n",
    "print(\"  Num examples = %d\"%(len(tr_inputs)))\n",
    "print(\"  Batch size = %d\"%(batch_num))\n",
    "print(\"  Num steps = %d\"%(num_train_optimization_steps))\n",
    "for _ in trange(epochs,desc=\"Epoch\"):\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        \n",
    "        # forward pass\n",
    "        outputs = model(b_input_ids, token_type_ids=None,\n",
    "        attention_mask=b_input_mask, labels=b_labels)\n",
    "        loss, scores = outputs[:2]\n",
    "        if n_gpu>1:\n",
    "            # When multi gpu, average it\n",
    "            loss = loss.mean()\n",
    "        \n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        \n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        \n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        \n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_out_address = 'models/bert_out_model/en09'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make dir if not exits\n",
    "if not os.path.exists(bert_out_address):\n",
    "        os.makedirs(bert_out_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model, configuration and tokenizer\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If we save using the predefined names, we can load using `from_pretrained`\n",
    "output_model_file = os.path.join(bert_out_address, \"pytorch_model.bin\")\n",
    "output_config_file = os.path.join(bert_out_address, \"config.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('models/bert_out_model/en09/vocab.txt',)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save model into file\n",
    "torch.save(model_to_save.state_dict(), output_model_file)\n",
    "model_to_save.config.to_json_file(output_config_file)\n",
    "tokenizer.save_vocabulary(bert_out_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0107 07:14:20.787202 139764591216384 configuration_utils.py:149] loading configuration file models/bert_out_model/en09/config.json\n",
      "I0107 07:14:20.789373 139764591216384 configuration_utils.py:169] Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 20,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0107 07:14:20.791067 139764591216384 modeling_utils.py:384] loading weights file models/bert_out_model/en09/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "model = BertForTokenClassification.from_pretrained(bert_out_address,num_labels=len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set model to GPU\n",
    "model.cuda();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "if n_gpu >1:\n",
    "    model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eval model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evalue loop\n",
    "model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running evaluation *****\n",
      "  Num examples =14388\n",
      "  Batch size = 32\n",
      "f1 socre: 0.835027\n",
      "Accuracy score: 0.971162\n",
      "***** Eval results *****\n",
      "\n",
      "           precision    recall  f1-score   support\n",
      "\n",
      "      tim     0.8549    0.8797    0.8671      6016\n",
      "      art     0.2342    0.1985    0.2149       131\n",
      "      gpe     0.9724    0.9333    0.9525      4830\n",
      "      geo     0.8260    0.9204    0.8707     11066\n",
      "      eve     0.3628    0.4505    0.4020        91\n",
      "      per     0.7908    0.7650    0.7777      5123\n",
      "      org     0.7415    0.6774    0.7080      5954\n",
      "      nat     0.4231    0.4490    0.4356        49\n",
      "\n",
      "micro avg     0.8276    0.8426    0.8350     33260\n",
      "macro avg     0.8278    0.8426    0.8339     33260\n",
      "\n",
      "f1 socre: 0.835027\n",
      "Accuracy score: 0.971162\n"
     ]
    }
   ],
   "source": [
    "eval_loss, eval_accuracy = 0, 0\n",
    "nb_eval_steps, nb_eval_examples = 0, 0\n",
    "y_true = []\n",
    "y_pred = []\n",
    "\n",
    "print(\"***** Running evaluation *****\")\n",
    "print(\"  Num examples ={}\".format(len(val_inputs)))\n",
    "print(\"  Batch size = {}\".format(batch_num))\n",
    "for step, batch in enumerate(valid_dataloader):\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "    input_ids, input_mask, label_ids = batch\n",
    "    \n",
    "#     if step > 2:\n",
    "#         break\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        outputs = model(input_ids, token_type_ids=None,\n",
    "        attention_mask=input_mask,)\n",
    "        # For eval mode, the first result of outputs is logits\n",
    "        logits = outputs[0] \n",
    "    \n",
    "    # Get NER predict result\n",
    "    logits = torch.argmax(F.log_softmax(logits,dim=2),dim=2)\n",
    "    logits = logits.detach().cpu().numpy()\n",
    "    \n",
    "    \n",
    "    # Get NER true result\n",
    "    label_ids = label_ids.to('cpu').numpy()\n",
    "    \n",
    "    \n",
    "    # Only predict the real word, mark=0, will not calculate\n",
    "    input_mask = input_mask.to('cpu').numpy()\n",
    "    \n",
    "    # Compare the valuable predict result\n",
    "    for i,mask in enumerate(input_mask):\n",
    "        # Real one\n",
    "        temp_1 = []\n",
    "        # Predict one\n",
    "        temp_2 = []\n",
    "        \n",
    "        for j, m in enumerate(mask):\n",
    "            # Mark=0, meaning its a pad word, dont compare\n",
    "            if m:\n",
    "                if tag2name[label_ids[i][j]] != \"X\" and tag2name[label_ids[i][j]] != \"[CLS]\" and tag2name[label_ids[i][j]] != \"[SEP]\" : # Exclude the X label\n",
    "                    temp_1.append(tag2name[label_ids[i][j]])\n",
    "                    temp_2.append(tag2name[logits[i][j]])\n",
    "            else:\n",
    "                break\n",
    "        \n",
    "            \n",
    "        y_true.append(temp_1)\n",
    "        y_pred.append(temp_2)\n",
    "\n",
    "        \n",
    "\n",
    "print(\"f1 socre: %f\"%(f1_score(y_true, y_pred)))\n",
    "print(\"Accuracy score: %f\"%(accuracy_score(y_true, y_pred)))\n",
    "\n",
    "# Get acc , recall, F1 result report\n",
    "report = classification_report(y_true, y_pred,digits=4)\n",
    "\n",
    "# Save the report into file\n",
    "output_eval_file = os.path.join(bert_out_address, \"eval_results.txt\")\n",
    "with open(output_eval_file, \"w\") as writer:\n",
    "    print(\"***** Eval results *****\")\n",
    "    print(\"\\n%s\"%(report))\n",
    "    print(\"f1 socre: %f\"%(f1_score(y_true, y_pred)))\n",
    "    print(\"Accuracy score: %f\"%(accuracy_score(y_true, y_pred)))\n",
    "    \n",
    "    writer.write(\"f1 socre:\\n\")\n",
    "    writer.write(str(f1_score(y_true, y_pred)))\n",
    "    writer.write(\"\\n\\nAccuracy score:\\n\")\n",
    "    writer.write(str(accuracy_score(y_true, y_pred)))\n",
    "    writer.write(\"\\n\\n\")  \n",
    "    writer.write(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After we trained a model, we can make it into service ---- sending a new sentence to the model,then get the prediction."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The process contains:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Load model\n",
    "- Load tokenizer\n",
    "- Set test query\n",
    "- Make query into embedding\n",
    "- Predict with model\n",
    "- Parser result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set tag index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tag to index, must be the same as we training\n",
    "tag2idx={'B-art': 14,\n",
    " 'B-eve': 16,\n",
    " 'B-geo': 0,\n",
    " 'B-gpe': 13,\n",
    " 'B-nat': 12,\n",
    " 'B-org': 10,\n",
    " 'B-per': 4,\n",
    " 'B-tim': 2,\n",
    " 'I-art': 5,\n",
    " 'I-eve': 7,\n",
    " 'I-geo': 15,\n",
    " 'I-gpe': 8,\n",
    " 'I-nat': 11,\n",
    " 'I-org': 3,\n",
    " 'I-per': 6,\n",
    " 'I-tim': 1,\n",
    " 'X':17,\n",
    " 'O': 9,\n",
    " '[CLS]':18,\n",
    " '[SEP]':19}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mapping index to name\n",
    "tag2name={tag2idx[key] : key for key in tag2idx.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model we trained before, the dir containing pytorch_model.bin and vocab.txt\n",
    "save_model_address = 'models/bert_out_model/en09'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0507 16:25:01.311969 139888124225280 configuration_utils.py:254] loading configuration file models/bert_out_model/en09/config.json\n",
      "I0507 16:25:01.313669 139888124225280 configuration_utils.py:292] Model config BertConfig {\n",
      "  \"architectures\": null,\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"do_sample\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\",\n",
      "    \"3\": \"LABEL_3\",\n",
      "    \"4\": \"LABEL_4\",\n",
      "    \"5\": \"LABEL_5\",\n",
      "    \"6\": \"LABEL_6\",\n",
      "    \"7\": \"LABEL_7\",\n",
      "    \"8\": \"LABEL_8\",\n",
      "    \"9\": \"LABEL_9\",\n",
      "    \"10\": \"LABEL_10\",\n",
      "    \"11\": \"LABEL_11\",\n",
      "    \"12\": \"LABEL_12\",\n",
      "    \"13\": \"LABEL_13\",\n",
      "    \"14\": \"LABEL_14\",\n",
      "    \"15\": \"LABEL_15\",\n",
      "    \"16\": \"LABEL_16\",\n",
      "    \"17\": \"LABEL_17\",\n",
      "    \"18\": \"LABEL_18\",\n",
      "    \"19\": \"LABEL_19\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_10\": 10,\n",
      "    \"LABEL_11\": 11,\n",
      "    \"LABEL_12\": 12,\n",
      "    \"LABEL_13\": 13,\n",
      "    \"LABEL_14\": 14,\n",
      "    \"LABEL_15\": 15,\n",
      "    \"LABEL_16\": 16,\n",
      "    \"LABEL_17\": 17,\n",
      "    \"LABEL_18\": 18,\n",
      "    \"LABEL_19\": 19,\n",
      "    \"LABEL_2\": 2,\n",
      "    \"LABEL_3\": 3,\n",
      "    \"LABEL_4\": 4,\n",
      "    \"LABEL_5\": 5,\n",
      "    \"LABEL_6\": 6,\n",
      "    \"LABEL_7\": 7,\n",
      "    \"LABEL_8\": 8,\n",
      "    \"LABEL_9\": 9\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 20,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "I0507 16:25:01.316571 139888124225280 modeling_utils.py:459] loading weights file models/bert_out_model/en09/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "save_model = BertForTokenClassification.from_pretrained(save_model_address,num_labels=len(tag2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "I0507 16:25:05.673797 139888124225280 tokenization_utils.py:417] Model name 'models/bert_out_model/en09' not found in model shortcut name list (bert-base-chinese, bert-base-finnish-cased-v1, bert-base-german-dbmdz-uncased, bert-large-cased-whole-word-masking-finetuned-squad, bert-large-cased, bert-base-uncased, bert-large-uncased-whole-word-masking, bert-base-multilingual-cased, bert-base-dutch-cased, bert-base-finnish-uncased-v1, bert-base-multilingual-uncased, bert-large-uncased-whole-word-masking-finetuned-squad, bert-base-german-dbmdz-cased, bert-base-cased, bert-large-uncased, bert-base-cased-finetuned-mrpc, bert-large-cased-whole-word-masking, bert-base-german-cased). Assuming 'models/bert_out_model/en09' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "I0507 16:25:05.675888 139888124225280 tokenization_utils.py:446] Didn't find file models/bert_out_model/en09/added_tokens.json. We won't load it.\n",
      "I0507 16:25:05.677664 139888124225280 tokenization_utils.py:446] Didn't find file models/bert_out_model/en09/tokenizer_config.json. We won't load it.\n",
      "I0507 16:25:05.679970 139888124225280 tokenization_utils.py:446] Didn't find file models/bert_out_model/en09/special_tokens_map.json. We won't load it.\n",
      "I0507 16:25:05.681512 139888124225280 tokenization_utils.py:499] loading file None\n",
      "I0507 16:25:05.682542 139888124225280 tokenization_utils.py:499] loading file None\n",
      "I0507 16:25:05.691070 139888124225280 tokenization_utils.py:499] loading file models/bert_out_model/en09/vocab.txt\n",
      "I0507 16:25:05.692115 139888124225280 tokenization_utils.py:499] loading file None\n"
     ]
    }
   ],
   "source": [
    "# Here, our save model address containing pytorch_model.bin and vocab.txt\n",
    "# So, we can load the tokenzier from the same dir as the save model address\n",
    "tokenizer = BertTokenizer.from_pretrained(save_model_address,do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set max sentence length, must be the same as our training process\n",
    "max_len  = 45"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set test query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_query = \"I live in USA, this is my IBM laptop.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make query into embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- token id embedding, need to tokenize first\n",
    "- mask word embedding\n",
    "- segmentation embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_texts = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_token = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add [CLS] at the front \n",
    "temp_token.append('[CLS]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_list = tokenizer.tokenize(test_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'live', 'in', 'USA', ',', 'this', 'is', 'my', 'IBM', 'laptop', '.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m,token in enumerate(token_list):\n",
    "    temp_token.append(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trim the token to fit the length requirement\n",
    "if len(temp_token) > max_len-1:\n",
    "    temp_token= temp_token[:max_len-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add [SEP] at the end\n",
    "temp_token.append('[SEP]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'I',\n",
       " 'live',\n",
       " 'in',\n",
       " 'USA',\n",
       " ',',\n",
       " 'this',\n",
       " 'is',\n",
       " 'my',\n",
       " 'IBM',\n",
       " 'laptop',\n",
       " '.',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_texts.append(temp_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make id embedding**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  101   146  1686  1107  3066   117  1142  1110  1139  9768 12574   119\n",
      "   102     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0]\n"
     ]
    }
   ],
   "source": [
    "# Make text token into id\n",
    "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                          maxlen=max_len, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
    "print(input_ids[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make mask embedding**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For fine tune of predict, with token mask is 1,pad token is 0\n",
    "attention_masks = [[int(i>0) for i in ii] for ii in input_ids]\n",
    "attention_masks[0];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make segmention type embedding**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "segment_ids = [[0] * len(input_id) for input_id in input_ids]\n",
    "segment_ids[0];"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make embeddings into torch tensor**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ids = torch.tensor(input_ids)\n",
    "attention_masks = torch.tensor(attention_masks)\n",
    "segment_ids = torch.tensor(segment_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict with model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We only send input_ids embedding to model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set save model to Evalue loop\n",
    "save_model.eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get model predict result\n",
    "with torch.no_grad():\n",
    "        outputs = save_model(input_ids, token_type_ids=None,\n",
    "        attention_mask=None,)\n",
    "        # For eval mode, the first result of outputs is logits\n",
    "        logits = outputs[0] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make logits into numpy type predict result\n",
    "# The predict result contain each token's all tags predict result\n",
    "predict_results = logits.detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 45, 20)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_results.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Make each token predict result into softmax mode**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.special import softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_arrays_soft = softmax(predict_results[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.9468857e-07, 6.0080066e-07, 4.0490704e-07, 4.3075502e-07,\n",
       "       5.3710983e-07, 8.1970501e-07, 4.4107060e-07, 6.8387163e-07,\n",
       "       1.4901701e-06, 4.3826424e-07, 4.2638061e-07, 1.4972500e-06,\n",
       "       1.3224795e-06, 3.3777670e-07, 2.8167085e-06, 4.8201440e-07,\n",
       "       1.5525582e-06, 6.1996315e-07, 8.1188858e-01, 1.1874760e-06],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_arrays_soft[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_array = result_arrays_soft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45, 20)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result_array),len(result_array[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parser result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make the predict array into readable result "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get each token final predict tag index result\n",
    "result_list = np.argmax(result_array,axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([18,  9,  9,  9,  0,  9,  9,  9,  9, 10,  9,  9, 19,  9, 17, 17, 17,\n",
       "       17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,\n",
       "       17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Get token predict tag**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token:[CLS]\n",
      "Predict_Tag:[CLS]\n",
      "\n",
      "Token:I\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:live\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:in\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:USA\n",
      "Predict_Tag:B-geo\n",
      "\n",
      "Token:,\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:this\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:is\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:my\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:IBM\n",
      "Predict_Tag:B-org\n",
      "\n",
      "Token:laptop\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:.\n",
      "Predict_Tag:O\n",
      "\n",
      "Token:[SEP]\n",
      "Predict_Tag:[SEP]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i, mark in enumerate(attention_masks[0]):\n",
    "    if mark>0:\n",
    "        print(\"Token:%s\"%(temp_token[i]))\n",
    "#         print(\"Tag:%s\"%(result_list[i]))\n",
    "        print(\"Predict_Tag:%s\"%(tag2name[result_list[i]]))\n",
    "        #print(\"Posibility:%f\"%(result_array[i][result_list[i]]))\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}