nlp-tasks/tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"bert-base-uncased\"\n",
    "#tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_examples = [\"today is not so bad\", \"It is so bad\", \"It's good\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 认识Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizer(name_or_path='bert-base-uncased', vocab_size=30522, model_max_length=512, is_fast=False, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=True)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'unk_token': '[UNK]',\n",
       " 'sep_token': '[SEP]',\n",
       " 'pad_token': '[PAD]',\n",
       " 'cls_token': '[CLS]',\n",
       " 'mask_token': '[MASK]'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.special_tokens_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[100, 102, 0, 101, 103]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_tokens_to_ids(tokenizer.special_tokens_map.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基本用法\n",
    "\n",
    "1. 分词\n",
    "2. token转为id\n",
    "3. 加特殊token、截断、补白\n",
    "4. 转为tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Keyword arguments {'max_lenght': 32} not recognized.\n",
      "Keyword arguments {'max_lenght': 32} not recognized.\n",
      "Keyword arguments {'max_lenght': 32} not recognized.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])\n",
      "tensor([[ 101, 2651, 2003, 2025, 2061, 2919,  102],\n",
      "        [ 101, 2009, 2003, 2061, 2919,  102,    0],\n",
      "        [ 101, 2009, 1005, 1055, 2204,  102,    0]])\n",
      "tensor([[1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 0],\n",
      "        [1, 1, 1, 1, 1, 1, 0]])\n"
     ]
    }
   ],
   "source": [
    "in_tensors = tokenizer(\n",
    "    test_examples, padding=True, truncation=True, max_lenght=32, return_tensors=\"pt\"\n",
    ")\n",
    "print(in_tensors.keys())\n",
    "print(in_tensors[\"input_ids\"])\n",
    "print(in_tensors[\"attention_mask\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分词： tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['today', 'is', 'not', 'so', 'bad'], ['it', 'is', 'so', 'bad'], ['it', \"'\", 's', 'good']]\n"
     ]
    }
   ],
   "source": [
    "tokens = [tokenizer.tokenize(text) for text in test_examples]\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 中文支持很有限"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[UNK] | [UNK] | ， | 中 | 国 | ， | bert | [UNK] | 中 | 文 | 的 | [UNK] | [UNK] | [UNK] | 有 | [UNK]'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' | '.join(tokenizer.tokenize(\"你好，中国，Bert对中文的支持很有限\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hello | - | cat | ! | , | ks | ##dh | ##12 | ##23 | , | 123 | ##45 | ##6'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' | '.join(tokenizer.tokenize(\"hello-cat!, ksdh1223, 123456\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 对于一些生僻，错误的word，会进行拆为wordpiece"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 转为id： convert_tokens_to_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2651, 2003, 2025, 2061, 2919], [2009, 2003, 2061, 2919], [2009, 1005, 1055, 2204]]\n"
     ]
    }
   ],
   "source": [
    "ids = [tokenizer.convert_tokens_to_ids(token) for token in tokens]\n",
    "print(ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `encode`\n",
    "\n",
    "encode = tokenize + convert_tokens_to_ids +  add special_token\n",
    "\n",
    "encode_plus = tokenize + convert_tokens_to_ids +  add special_token,  generate mask\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[101, 2651, 2003, 2025, 2061, 2919, 102]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.encode(test_examples[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]', 'today', 'is', 'not', 'so', 'bad', '[SEP]']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.convert_ids_to_tokens(tokenizer.encode(test_examples[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `encode_plus` 编码一对句子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('input_ids', [101, 2651, 2003, 2025, 2061, 2919, 102, 2009, 2003, 2061, 2919, 102])\n",
      "('token_type_ids', [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1])\n",
      "('attention_mask', [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])\n"
     ]
    }
   ],
   "source": [
    "ids = tokenizer.encode_plus(test_examples[0], test_examples[1])\n",
    "print(*ids.items(), sep='\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "token_type_ids中0表示第一个句子的id，1表示第2个句子的id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# decode\n",
    "\n",
    "decode是encode的逆运算：将id list 转化为一个字符串"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS] today is not so bad [SEP] it is so bad [SEP]'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(ids[\"input_ids\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fast Tokenizer / Slow Tokenizer\n",
    "\n",
    "Fast Tokenizer 是基于rust来实现的，速度快；而Slow tokenizer是基于python实现，速度慢；"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fast_tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "slow_tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "122 ms ± 3.33 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "for _ in range(1000):\n",
    "    fast_tokenizer(test_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "419 ms ± 8.19 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "for _ in range(1000):\n",
    "    slow_tokenizer(test_examples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FastTokenizer有一些特殊的返回值\n",
    "\n",
    "* offset_mapping：标记了每一个token在原输出str中的索引位置\n",
    "* word_ids：标记了每个token对应原输出中word的索引\n",
    "\n",
    "这个对于NER或QA来说比较重要。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 1999, 1996, 2502, 2502, 2088, 1010, 1045, 2031, 1037, 2502, 3959, 6562, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'offset_mapping': [(0, 0), (0, 2), (3, 6), (7, 10), (11, 14), (15, 20), (20, 21), (22, 23), (24, 28), (29, 30), (31, 34), (35, 40), (40, 44), (0, 0)]}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = fast_tokenizer(\"In the big big world, I have a big dreamming\", return_offsets_mapping=True)\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[CLS]', 'in', 'the', 'big', 'big', 'world', ',', 'i', 'have', 'a', 'big', 'dream', '##ming', '[SEP]']\n"
     ]
    }
   ],
   "source": [
    "print(fast_tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, None]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs.word_ids()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "d2l",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}