NLP_100_9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wy4FSmYcJIwK"
   },
   "source": [
    "# RNN, CNN\n",
    "## https://nlp100.github.io/ja/ch09.html\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "67ddAau-HU9t"
   },
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/iamtatsuki05/NLP_100/blob/fix_all_merge/NLP_100_9.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jBYlYY_Q7o3R"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "b3Tng2hjetMB"
   },
   "outputs": [],
   "source": [
    "!pip3 install -U polars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CJIjsxfDJcfv"
   },
   "source": [
    "# ID番号への変換\n",
    "## 問題51で構築した学習データ中の単語にユニークなID番号を付与したい．学習データ中で最も頻出する単語に1，2番目に頻出する単語に2，……といった方法で，学習データ中で2回以上出現する単語にID番号を付与せよ．そして，与えられた単語列に対して，ID番号の列を返す関数を実装せよ．ただし，出現頻度が2回未満の単語のID番号はすべて0とせよ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "03YyU3nTKzhG"
   },
   "outputs": [],
   "source": [
    "#50,51をもう一度\n",
    "!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00359/NewsAggregatorDataset.zip\n",
    "!unzip NewsAggregatorDataset.zip\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# df = pd.read_csv('/content/newsCorpora.csv', header=None, sep='\\t', names=['ID', 'TITLE', 'URL', 'PUBLISHER', 'CATEGORY', 'STORY', 'HOSTNAME', 'TIMESTAMP'])\n",
    "df = pl.read_csv('/content/newsCorpora.csv',\n",
    "                 has_header=False,\n",
    "                 separator='\\t',\n",
    "                 ignore_errors=True,\n",
    "                 encoding=\"utf8\",\n",
    "                 new_columns=['ID', 'TITLE', 'URL', 'PUBLISHER', 'CATEGORY', 'STORY', 'HOSTNAME', 'TIMESTAMP'],\n",
    "                 )\n",
    "# df = df.loc[df['PUBLISHER'].isin(['Reuters', 'Huffington Post', 'Businessweek', 'Contactmusic.com', 'Daily Mail']), ['TITLE', 'CATEGORY']]\n",
    "df = df.filter(df['PUBLISHER'].is_in(['Reuters', 'Huffington Post', 'Businessweek', 'Contactmusic.com', 'Daily Mail'])).select(['TITLE', 'CATEGORY'])\n",
    "\n",
    "# データの分割\n",
    "test, train_valid = train_test_split(df, test_size=0.8, shuffle=True, random_state=42, stratify=df['CATEGORY'])\n",
    "valid, train = train_test_split(train_valid, test_size=0.25, shuffle=True, random_state=42, stratify=train_valid['CATEGORY'])\n",
    "# train.reset_index(drop=True, inplace=True)\n",
    "# valid.reset_index(drop=True, inplace=True)\n",
    "# test.reset_index(drop=True, inplace=True)\n",
    "train = train.with_row_count()\n",
    "valid = valid.with_row_count()\n",
    "test = test.with_row_count()\n",
    "\n",
    "from collections import defaultdict\n",
    "import string\n",
    "\n",
    "d = defaultdict(int)\n",
    "table = str.maketrans(string.punctuation, ' '*len(string.punctuation))#記号処理\n",
    "for text in train['TITLE']:\n",
    "    for word in text.translate(table).split():\n",
    "        d[word] += 1\n",
    "d = sorted(d.items(), key=lambda x:x[1], reverse=True)#sort\n",
    "\n",
    "word_id = {word: idx + 1 for idx, (word, num) in enumerate(d) if num > 1}#辞書\n",
    "word_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xyOkRJSIcAxp"
   },
   "outputs": [],
   "source": [
    "PAD_TOKEN = '<PAD>'\n",
    "UNK_TOKEN = '<UNK>'\n",
    "PAD = 0 \n",
    "UNK = 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ozN7-AMWcCkv"
   },
   "outputs": [],
   "source": [
    "word2id = {\n",
    "    PAD_TOKEN: PAD,\n",
    "    UNK_TOKEN: UNK,\n",
    "}\n",
    "\n",
    "MIN_COUNT = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YOGqeRz6cKgB"
   },
   "outputs": [],
   "source": [
    "class Vocab(object):\n",
    "\n",
    "    def __init__(self, word2id={}):\n",
    "        self.word2id = dict(word2id)\n",
    "        self.id2word = {v: k for k, v in self.word2id.items()}    \n",
    "\n",
    "    def build_vocab(self, sentences, min_count=1):\n",
    "        word_counter = {}\n",
    "        for sentence in sentences:\n",
    "            for word in sentence:\n",
    "\n",
    "                word_counter[word] = word_counter.get(word, 0) + 1\n",
    "        for word, count in sorted(word_counter.items(), key=lambda x: -x[1]):\n",
    "            if count < min_count:\n",
    "                break\n",
    "            _id = len(self.word2id)\n",
    "            self.word2id.setdefault(word, _id)\n",
    "            self.id2word[_id] = word\n",
    "        self.raw_vocab = {w: word_counter[w] for w in self.word2id.keys() if w in word_counter}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0A5byM-PcTfm"
   },
   "outputs": [],
   "source": [
    "vocab = Vocab(word2id=word2id)\n",
    "vocab.build_vocab(train, min_count=MIN_COUNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hbZjB-bZcVTh"
   },
   "outputs": [],
   "source": [
    "def convert_sentence_to_ids(vocab, sen):\n",
    "    result = [vocab.word2id.get(word, UNK) for word in sen]\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5EZMqwZzcY8O"
   },
   "outputs": [],
   "source": [
    "id_train = [convert_sentence_to_ids(vocab, sen) for sen in train]\n",
    "print(id_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5xPslxxUdpj9"
   },
   "outputs": [],
   "source": [
    "vocab.build_vocab(valid, min_count=MIN_COUNT)\n",
    "id_valid = [convert_sentence_to_ids(vocab, sen) for sen in valid]\n",
    "vocab.build_vocab(test, min_count=MIN_COUNT)\n",
    "id_test = [convert_sentence_to_ids(vocab, sen) for sen in test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LuJtaSjy4j8Q"
   },
   "outputs": [],
   "source": [
    "# from collections import Counter\n",
    "\n",
    "# counter = Counter([\n",
    "#     x\n",
    "#     for sent in train\n",
    "#     for x in sent\n",
    "# ])\n",
    "# vocab_in_train = [\n",
    "#     token\n",
    "#     for token, freq in counter.most_common()\n",
    "#     if freq > 1\n",
    "# ]\n",
    "# vocab_list = ['[UNK]'] + vocab_in_train\n",
    "# vocab_dict = {x:n for n, x in enumerate(vocab_list)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4T0JM2tp5eWx"
   },
   "outputs": [],
   "source": [
    "# def sent_to_ids(sent):\n",
    "#     return torch.tensor([vocab_dict[x if x in vocab_dict else '[UNK]'] for x in sent], dtype=torch.long)\n",
    "# def dataset_to_ids(dataset):\n",
    "#     return [sent_to_ids(x) for x in dataset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VRFPOGPC5lTV"
   },
   "outputs": [],
   "source": [
    "# import torch\n",
    "# train_ds = dataset_to_ids(train)\n",
    "# valid_ds = dataset_to_ids(valid)\n",
    "# test_ds = dataset_to_ids(test)\n",
    "# train_ds[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RUHR6GfrJcia"
   },
   "source": [
    "# RNNによる予測\n",
    "## ID番号で表現された単語列x=(x1,x2,…,xT)がある．ただし，Tは単語列の長さ，xt∈ℝVは単語のID番号のone-hot表記である（Vは単語の総数である）．再帰型ニューラルネットワーク（RNN: Recurrent Neural Network）を用い，単語列xからカテゴリyを予測するモデルとして，次式を実装せよ．\n",
    "\n",
    "h→0=0,h→t=RNN−→−−(emb(xt),h→t−1),y=softmax(W(yh)h→T+b(y))\n",
    "ただし，emb(x)∈ℝdwは単語埋め込み（単語のone-hot表記から単語ベクトルに変換する関数），h→t∈ℝdhは時刻tの隠れ状態ベクトル，RNN−→−−(x,h)は入力xと前時刻の隠れ状態hから次状態を計算するRNNユニット，W(yh)∈ℝL×dhは隠れ状態ベクトルからカテゴリを予測するための行列，b(y)∈ℝLはバイアス項である（dw,dh,Lはそれぞれ，単語埋め込みの次元数，隠れ状態ベクトルの次元数，ラベル数である）．RNNユニットRNN−→−−(x,h)には様々な構成が考えられるが，典型例として次式が挙げられる．\n",
    "\n",
    "RNN−→−−(x,h)=g(W(hx)x+W(hh)h+b(h))\n",
    "ただし，W(hx)∈ℝdh×dw，W(hh)∈ℝdh×dh,b(h)∈ℝdhはRNNユニットのパラメータ，gは活性化関数（例えばtanhやReLUなど）である．\n",
    "\n",
    "なお，この問題ではパラメータの学習を行わず，ランダムに初期化されたパラメータでyを計算するだけでよい．次元数などのハイパーパラメータは，dw=300,dh=50など，適当な値に設定せよ（以降の問題でも同様である）．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HZAWporJ92Nc"
   },
   "outputs": [],
   "source": [
    "#dw=300, dh=50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cmQohg-guRUT"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "torch.manual_seed(42)\n",
    "\n",
    "class RNN(nn.Module):\n",
    "    def __init__(self, vocab_size, emb_size, padding_idx, output_size, hidden_size):\n",
    "        super().__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.emb = nn.Embedding(vocab_size, emb_size, padding_idx=padding_idx)\n",
    "        self.rnn = nn.RNN(emb_size, hidden_size, nonlinearity='relu', batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size, output_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        self.batch_size = x.size()[0]\n",
    "        hidden = self.init_hidden()\n",
    "        emb = self.emb(x)\n",
    "        out, hidden = self.rnn(emb, hidden)\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out\n",
    "        \n",
    "    def init_hidden(self):\n",
    "        hidden = torch.zeros(1, self.batch_size, self.hidden_size)\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_YJxuPDht3Zv"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "#テキスト修正\n",
    "def tokenizer(text, word2id=word_id, unk = 0):\n",
    "    table = str.maketrans(string.punctuation, ' '*len(string.punctuation))\n",
    "    return [word2id.get(word, unk) for word in text.translate(table).split()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "UDZqdi2OuUjb"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset\n",
    "\n",
    "class NewsCorporaDataset(Dataset):\n",
    "    def __init__(self, X, y, tokenizer):\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        text = self.X[index]\n",
    "        inputs = self.tokenizer(text)\n",
    "\n",
    "        return {'inputs': torch.tensor(inputs, dtype=torch.int64), 'labels': torch.tensor(self.y[index], dtype=torch.int64)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "EaTS841ruWCD"
   },
   "outputs": [],
   "source": [
    "# ラベルベクトル\n",
    "category_dict = {'b': 0, 't': 1, 'e':2, 'm':3}\n",
    "# y_train = train['CATEGORY'].map(lambda x: category_dict[x]).values\n",
    "# y_valid = valid['CATEGORY'].map(lambda x: category_dict[x]).values\n",
    "# y_test = test['CATEGORY'].map(lambda x: category_dict[x]).values\n",
    "y_train = train['CATEGORY'].apply(lambda x: category_dict[x])\n",
    "y_valid = valid['CATEGORY'].apply(lambda x: category_dict[x])\n",
    "y_test = test['CATEGORY'].apply(lambda x: category_dict[x])\n",
    "\n",
    "dataset_train = NewsCorporaDataset(train['TITLE'], y_train, tokenizer)\n",
    "dataset_valid = NewsCorporaDataset(valid['TITLE'], y_valid, tokenizer)\n",
    "dataset_test = NewsCorporaDataset(test['TITLE'], y_test, tokenizer)\n",
    "\n",
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "hidden_size = 50\n",
    "\n",
    "model = RNN(vocab_size, emb_size, padding_idx, output_size, hidden_size)\n",
    "\n",
    "# 先頭10件の予測値取得\n",
    "for num in range(10):\n",
    "  X = dataset_train[num]['inputs']\n",
    "  print(torch.softmax(model(X.unsqueeze(0)), dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9xFNbtAhKyyt"
   },
   "outputs": [],
   "source": [
    "# 参考https://exture-ri.com/2021/01/12/pytorch-rnn/\n",
    "# https://gotutiyan.hatenablog.com/entry/2020/09/02/200144"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ybr_R3_0Jckm"
   },
   "source": [
    "# 確率的勾配降下法による学習\n",
    "## 確率的勾配降下法（SGD: Stochastic Gradient Descent）を用いて，問題81で構築したモデルを学習せよ．訓練データ上の損失と正解率，評価データ上の損失と正解率を表示しながらモデルを学習し，適当な基準（例えば10エポックなど）で終了させよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7Plx2-1lKyPm"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from torch import optim\n",
    "\n",
    "def calc_acc(model, dataset, device=None, criterion=None):\n",
    "    dataloader = DataLoader(dataset, batch_size=1, shuffle=False)\n",
    "    loss = 0.0\n",
    "    total = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data in dataloader:\n",
    "        \n",
    "            inputs = data['inputs'].to(device)#gpu\n",
    "            labels = data['labels'].to(device)#gpu\n",
    "            outputs = model(inputs)#計算\n",
    "            if criterion != None:\n",
    "                loss += criterion(outputs, labels).item()#loss\n",
    "                #正解率\n",
    "            pred = torch.argmax(outputs, dim=-1)\n",
    "            total += len(inputs)\n",
    "            correct += (pred == labels).sum().item()\n",
    "        \n",
    "    return loss / len(dataset), correct / total\n",
    "  \n",
    "\n",
    "def train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, collate_fn=None, device=None):\n",
    "    model.to(device)\n",
    "    dataloader_train = DataLoader(dataset_train, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)\n",
    "    dataloader_valid = DataLoader(dataset_valid, batch_size=1, shuffle=False)\n",
    "    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs, eta_min=1e-3, last_epoch=-1)\n",
    "    loss_train_list = []\n",
    "    loss_valid_list = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        model.train()\n",
    "        for data in dataloader_train:\n",
    "            optimizer.zero_grad()#初期化\n",
    "            inputs = data['inputs'].to(device)#gpu\n",
    "            labels = data['labels'].to(device)#gpu\n",
    "            #計算\n",
    "            outputs = model.forward(inputs)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()#調節\n",
    "            optimizer.step()#更新\n",
    "        \n",
    "        model.eval()\n",
    "\n",
    "        # 損失と正解率の算出\n",
    "        loss_train, acc_train = calc_acc(model, dataset_train, device, criterion=criterion)\n",
    "        loss_valid, acc_valid = calc_acc(model, dataset_valid, device, criterion=criterion)\n",
    "        loss_train_list.append([loss_train, acc_train])\n",
    "        loss_valid_list.append([loss_valid, acc_valid])\n",
    "\n",
    "        #パラメータ保存\n",
    "        torch.save({'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict()}, f'checkpoint{epoch + 1}.pt')\n",
    "\n",
    "        #正答率誤算などの算出\n",
    "        print(f'epoch: {epoch + 1}, loss_train: {loss_train:.4f}, acc_train: {acc_train:.4f}, loss_valid: {loss_valid:.4f}, acc_valid: {acc_valid:.4f}') \n",
    "        \n",
    "        scheduler.step()\n",
    "\n",
    "    return {'train': loss_train, 'valid': loss_valid}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "f5vSVETA2pL-"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "hidden_size = 50\n",
    "learning_rate = 1e-3\n",
    "batch_size = 1\n",
    "num_epochs = 3#testのため回数少なめ\n",
    "\n",
    "model = RNN(vocab_size, emb_size, padding_idx, output_size, hidden_size)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "log = train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HHAgvIA7VEPs"
   },
   "outputs": [],
   "source": [
    "# https://note.nkmk.me/python-pytorch-device-to-cuda-cpu/\n",
    "# https://runebook.dev/ja/docs/pytorch/generated/torch.nn.bcewithlogitsloss\n",
    "# https://pytorch.org/docs/stable/generated/torch.nn.BCEWithLogitsLoss.html\n",
    "# https://tips-memo.com/python-diff-bce"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v2PtIrKWJcmi"
   },
   "source": [
    "# ミニバッチ化・GPU上での学習\n",
    "## 問題82のコードを改変し，B事例ごとに損失・勾配を計算して学習を行えるようにせよ（Bの値は適当に選べ）．また，GPU上で学習を実行せよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XJZk_XGx7m87"
   },
   "outputs": [],
   "source": [
    "class PadSequence():\n",
    "    def __init__(self, padding_idx):\n",
    "        self.padding_idx = padding_idx\n",
    "\n",
    "    def __call__(self, batch):\n",
    "        sorted_batch = sorted(batch, key=lambda x: x['inputs'].shape[0], reverse=True)\n",
    "        sequences = [x['inputs'] for x in sorted_batch]\n",
    "        sequences_padded = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=True, padding_value=self.padding_idx)\n",
    "        labels = torch.LongTensor([x['labels'] for x in sorted_batch])\n",
    "\n",
    "        return {'inputs': sequences_padded, 'labels': labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1YYLgaRh8WpJ"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "hidden_size = 50\n",
    "learning_rate = 1e-3\n",
    "batch_size = 50\n",
    "num_epochs = 3#testのため回数少なめ\n",
    "\n",
    "model = RNN(vocab_size, emb_size, padding_idx, output_size, hidden_size)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate)\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "log = train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, collate_fn=PadSequence(padding_idx), device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "j8myAgh79Bn9"
   },
   "outputs": [],
   "source": [
    "# 参考https://atmarkit.itmedia.co.jp/ait/articles/2008/28/news030.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-HVipbmRJcoz"
   },
   "source": [
    "# 単語ベクトルの導入\n",
    "## 事前学習済みの単語ベクトル（例えば，Google Newsデータセット（約1,000億単語）での学習済み単語ベクトル）で単語埋め込みemb(x)を初期化し，学習せよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "t9DUNeZMKxHc"
   },
   "outputs": [],
   "source": [
    "# # ダウンロード制限がかかっているのでprthを指定する\n",
    "# ! pip install --upgrade gdown\n",
    "# import gdown\n",
    "# gdown.download('https://drive.google.com/u/0/uc?id=0B7XkCwpI5KDYNlNUTTlSS21pQmM&export=download', './GoogleNews-vectors-negative300.bin.gz', quiet=False)\n",
    "# #model\n",
    "# from gensim.models import KeyedVectors\n",
    "\n",
    "# # 学習済みモデルのロード\n",
    "# model = KeyedVectors.load_word2vec_format('./GoogleNews-vectors-negative300.bin.gz' , binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YDSGA8QAVQt9"
   },
   "outputs": [],
   "source": [
    "# 直接pathを指定する場合\n",
    "from gensim.models import KeyedVectors\n",
    "model_kv = KeyedVectors.load_word2vec_format('/content/drive/MyDrive/Tutorial/GoogleNews-vectors-negative300.bin.gz', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DMMLdQGXtADB"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "weights = np.zeros((vocab_size , emb_size))\n",
    "wordataset_in_pretrained = 0\n",
    "for idx, word in enumerate(word_id.keys()):\n",
    "    if KeyError:\n",
    "        weights[idx] = np.random.normal(loc=0, scale=1, size=(emb_size,))#正規化\n",
    "    else:\n",
    "        weights[idx] = model_kv[word]\n",
    "        wordataset_in_pretrained += 1\n",
    "weights = torch.from_numpy(weights.astype((np.float32)))#torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Zm1HSzW6v_QK"
   },
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, vocab_size, emb_size, padding_idx, output_size, hidden_size, num_layers, emb_weights=None, bidirectional=False):\n",
    "        super().__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.num_directions = bidirectional + 1\n",
    "\n",
    "        if emb_weights != None:\n",
    "            self.emb = nn.Embedding.from_pretrained(emb_weights, padding_idx=padding_idx)\n",
    "        else:\n",
    "            self.emb = nn.Embedding(vocab_size, emb_size, padding_idx=padding_idx)\n",
    "\n",
    "        self.rnn = nn.RNN(emb_size, hidden_size, num_layers, nonlinearity='relu', bidirectional=bidirectional, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size * self.num_directions, output_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        self.batch_size = x.size()[0]\n",
    "        hidden = self.init_hidden()\n",
    "        emb = self.emb(x)\n",
    "        out, hidden = self.rnn(emb, hidden)\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out\n",
    "        \n",
    "    def init_hidden(self):\n",
    "        hidden = torch.zeros(self.num_layers * self.num_directions, self.batch_size, self.hidden_size)\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YhaFJc6NwB7h"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "hidden_size = 50\n",
    "num_layers = 1\n",
    "learning_rate = 1e-3\n",
    "batch_size = 50\n",
    "num_epochs = 3#testのため回数少なめ\n",
    "\n",
    "model = RNN(vocab_size, emb_size, padding_idx, output_size, hidden_size, num_layers, emb_weights=weights)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(model.parameters() , lr = learning_rate)\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "log = train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, collate_fn=PadSequence(padding_idx), device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "S7F_W9ROw4kM"
   },
   "outputs": [],
   "source": [
    "# 参考https://www.sejuku.net/blog/73026\n",
    "# https://note.nkmk.me/python-numpy-dtype-astype/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iVxcSaQHJcq3"
   },
   "source": [
    "# 双方向RNN・多層化\n",
    "## \n",
    "順方向と逆方向のRNNの両方を用いて入力テキストをエンコードし，モデルを学習せよ．\n",
    "\n",
    "h⃖ T+1=0,h⃖ t=RNN←−−−(emb(xt),h⃖ t+1),y=softmax(W(yh)[h→T;h⃖ 1]+b(y))\n",
    "ただし，h→t∈ℝdh,h⃖ t∈ℝdhはそれぞれ，順方向および逆方向のRNNで求めた時刻tの隠れ状態ベクトル，RNN←−−−(x,h)は入力xと次時刻の隠れ状態hから前状態を計算するRNNユニット，W(yh)∈ℝL×2dhは隠れ状態ベクトルからカテゴリを予測するための行列，b(y)∈ℝLはバイアス項である．また，[a;b]はベクトルaとbの連結を表す。\n",
    "\n",
    "さらに，双方向RNNを多層化して実験せよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z5P4B_uEKwSJ"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "hidden_size = 50\n",
    "num_layers = 5#ここで多層化\n",
    "learning_rate = 1e-3\n",
    "batch_size = 50\n",
    "num_epochs = 3#testのため回数少なめ\n",
    "\n",
    "model = RNN(vocab_size, emb_size, padding_idx, output_size, hidden_size, num_layers, emb_weights=weights, bidirectional=True)#bidirectional = Trueで双方向\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "log = train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, collate_fn=PadSequence(padding_idx), device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4orHQA2b2hR9"
   },
   "outputs": [],
   "source": [
    "# 参考https://qiita.com/m__k/items/78a5125d719951ca98d3\n",
    "# https://axa.biopapyrus.jp/deep-learning/rnn/brnn.html\n",
    "# https://deepage.net/deep_learning/2017/05/23/recurrent-neural-networks.html\n",
    "# https://teratail.com/questions/185713\n",
    "# https://qiita.com/tetsuro_skiing/items/87c0c37cefd7b601f974"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "wMhQ_gsMJctM"
   },
   "source": [
    "#  畳み込みニューラルネットワーク (CNN)\n",
    "## ID番号で表現された単語列x=(x1,x2,…,xT)がある．ただし，Tは単語列の長さ，xt∈ℝVは単語のID番号のone-hot表記である（Vは単語の総数である）．畳み込みニューラルネットワーク（CNN: Convolutional Neural Network）を用い，単語列xからカテゴリyを予測するモデルを実装せよ．\n",
    "\n",
    "ただし，畳み込みニューラルネットワークの構成は以下の通りとする．\n",
    "\n",
    "単語埋め込みの次元数: dw\n",
    "畳み込みのフィルターのサイズ: 3 トークン\n",
    "畳み込みのストライド: 1 トークン\n",
    "畳み込みのパディング: あり\n",
    "畳み込み演算後の各時刻のベクトルの次元数: dh\n",
    "畳み込み演算後に最大値プーリング（max pooling）を適用し，入力文をdh次元の隠れベクトルで表現\n",
    "すなわち，時刻tの特徴ベクトルpt∈ℝdhは次式で表される．\n",
    "\n",
    "pt=g(W(px)[emb(xt−1);emb(xt);emb(xt+1)]+b(p))\n",
    "ただし，W(px)∈ℝdh×3dw,b(p)∈ℝdhはCNNのパラメータ，gは活性化関数（例えばtanhやReLUなど），[a;b;c]はベクトルa,b,cの連結である．なお，行列W(px)の列数が3dwになるのは，3個のトークンの単語埋め込みを連結したものに対して，線形変換を行うためである．\n",
    "\n",
    "最大値プーリングでは，特徴ベクトルの次元毎に全時刻における最大値を取り，入力文書の特徴ベクトルc∈ℝdhを求める．c[i]でベクトルcのi番目の次元の値を表すことにすると，最大値プーリングは次式で表される．\n",
    "\n",
    "c[i]=max1≤t≤Tpt[i]\n",
    "最後に，入力文書の特徴ベクトルcに行列W(yc)∈ℝL×dhとバイアス項b(y)∈ℝLによる線形変換とソフトマックス関数を適用し，カテゴリyを予測する．\n",
    "\n",
    "y=softmax(W(yc)c+b(y))\n",
    "なお，この問題ではモデルの学習を行わず，ランダムに初期化された重み行列でyを計算するだけでよい．\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1SC8JKlo9xXa"
   },
   "outputs": [],
   "source": [
    "# 単語埋め込みの次元数: dw\n",
    "# 畳み込みのフィルターのサイズ: 3 トークン\n",
    "# 畳み込みのストライド: 1 トークン\n",
    "# 畳み込みのパディング: あり\n",
    "# 畳み込み演算後の各時刻のベクトルの次元数: dh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lZ_UFaz-Kvo-"
   },
   "outputs": [],
   "source": [
    "from torch.nn import functional as F\n",
    "\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self, vocab_size, emb_size, padding_idx, output_size, out_channels, kernel_heights, stride, padding, emb_weights=None):\n",
    "        super().__init__()\n",
    "        if emb_weights != None:\n",
    "            self.emb = nn.Embedding.from_pretrained(emb_weights, padding_idx=padding_idx)\n",
    "        else:\n",
    "            self.emb = nn.Embedding(vocab_size, emb_size, padding_idx=padding_idx)\n",
    "        self.conv = nn.Conv2d(1, out_channels, (kernel_heights, emb_size), stride, (padding, 0))\n",
    "        self.drop = nn.Dropout(0.3)\n",
    "        self.fc = nn.Linear(out_channels, output_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        emb = self.emb(x).unsqueeze(1)\n",
    "        conv = self.conv(emb)\n",
    "        act = F.relu(conv.squeeze(3))\n",
    "        max_pool = F.max_pool1d(act, act.size()[2])\n",
    "        out = self.fc(self.drop(max_pool.squeeze(2)))\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "slTiYPPo--EV"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "#CNNのパラメータ\n",
    "out_channels =100\n",
    "kernel_heights = 3\n",
    "stride = 1\n",
    "padding = 1\n",
    "\n",
    "model = CNN(vocab_size, emb_size, padding_idx, output_size, out_channels, kernel_heights, stride, padding, emb_weights=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0mwy7kvC_vId"
   },
   "outputs": [],
   "source": [
    "for num in range(10):\n",
    "    X = dataset_train[num]['inputs']\n",
    "    print(torch.softmax(model(X.unsqueeze(0)), dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "jhOnOGt06Ery"
   },
   "outputs": [],
   "source": [
    "# 参考https://qiita.com/shu_marubo/items/70b20c3a6c172aaeb8de\n",
    "# https://qiita.com/mathlive/items/8e1f9a8467fff8dfd03c\n",
    "# https://exture-ri.com/2021/01/11/pytorch-cnn/\n",
    "# https://qiita.com/m__k/items/6c39cfe7dfa99102fa8e\n",
    "# https://kento1109.hatenablog.com/entry/2019/09/30/115139"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ac0_YmHBJcvP"
   },
   "source": [
    "# 確率的勾配降下法によるCNNの学習\n",
    "## 確率的勾配降下法（SGD: Stochastic Gradient Descent）を用いて，問題86で構築したモデルを学習せよ．訓練データ上の損失と正解率，評価データ上の損失と正解率を表示しながらモデルを学習し，適当な基準（例えば10エポックなど）で終了させよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7BoDooPEKvN1"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "#CNNのパラメータ\n",
    "out_channels =100\n",
    "kernel_heights = 3\n",
    "stride = 1\n",
    "padding = 1\n",
    "learning_rate = 1e-3\n",
    "batch_size = 50\n",
    "num_epochs = 3#testのため回数少なめ\n",
    "\n",
    "model = CNN(vocab_size, emb_size, padding_idx, output_size, out_channels, kernel_heights, stride, padding, emb_weights=weights)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "# モデルの学習\n",
    "log = train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, collate_fn=PadSequence(padding_idx), device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d_QtLlgYJcxQ"
   },
   "source": [
    "# パラメータチューニング\n",
    "## 問題85や問題87のコードを改変し，ニューラルネットワークの形状やハイパーパラメータを調整しながら，高性能なカテゴリ分類器を構築せよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PBgntiKP6C8C"
   },
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, vocab_size, emb_size, padding_idx, output_size, hidden_size, num_layers, emb_weights=None, bidirectional=False):\n",
    "        super().__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.num_directions = bidirectional + 1\n",
    "\n",
    "        if emb_weights != None:\n",
    "            self.emb = nn.Embedding.from_pretrained(emb_weights, padding_idx=padding_idx)\n",
    "        else:\n",
    "            self.emb = nn.Embedding(vocab_size, emb_size, padding_idx=padding_idx)\n",
    "\n",
    "        self.rnn = nn.RNN(emb_size, hidden_size, num_layers, nonlinearity='relu', bidirectional=bidirectional, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_size * self.num_directions, output_size)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        self.batch_size = x.size()[0]\n",
    "        hidden = self.init_hidden()\n",
    "        emb = self.emb(x)\n",
    "        out, hidden = self.rnn(emb, hidden)\n",
    "        out = self.fc(out[:, -1, :])\n",
    "        return out\n",
    "        \n",
    "    def init_hidden(self):\n",
    "        hidden = torch.zeros(self.num_layers * self.num_directions, self.batch_size, self.hidden_size)\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QudC7shH6L4r"
   },
   "outputs": [],
   "source": [
    "vocab_size = len(set(word_id.values())) + 1\n",
    "emb_size = 300\n",
    "padding_idx = len(set(word_id.values()))\n",
    "output_size = 4\n",
    "hidden_size = 50\n",
    "num_layers = 10#多い方が良い？\n",
    "learning_rate = 1e-3#小さい方がいい？\n",
    "batch_size = 128#適度に増やす\n",
    "num_epochs = 25#多めに\n",
    "\n",
    "model = RNN(vocab_size, emb_size, padding_idx, output_size, hidden_size, num_layers, emb_weights=weights, bidirectional=True)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "\n",
    "log = train_model(dataset_train, dataset_valid, batch_size, model, criterion, optimizer, num_epochs, collate_fn=PadSequence(padding_idx), device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pNYDnFE_EfYg"
   },
   "outputs": [],
   "source": [
    "# 参考https://qiita.com/nyanko-box/items/a6f50e28383a5bd0a432\n",
    "# https://cpp-learning.com/optuna-pytorch/\n",
    "# https://qiita.com/Yushi1958/items/cd22ade638f7e292e520\n",
    "# https://dreamer-uma.com/pytorch-optuna-hyperparameter-tuning/\n",
    "# http://maruo51.com/2020/08/07/optuna_pytorch/\n",
    "# https://ichi.pro/optuna-o-shiyoshita-pytorch-haipa-parame-ta-no-chosei-4883072668892"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "es6FbyNdJczi"
   },
   "source": [
    "# 事前学習済み言語モデルからの転移学習\n",
    "## 事前学習済み言語モデル（例えばBERTなど）を出発点として，ニュース記事見出しをカテゴリに分類するモデルを構築せよ．"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VlgIx1syKuK-"
   },
   "outputs": [],
   "source": [
    "# !pip install -q transformers\n",
    "# from transformers import BertTokenizer, BertModel\n",
    "# from torch import cuda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "rfL9OG1rEehF"
   },
   "outputs": [],
   "source": [
    "# https://note.nkmk.me/python-pytorch-device-to-cuda-cpu/\n",
    "# https://qiita.com/yamaru/items/63a342c844cff056a549\n",
    "# https://qiita.com/m__k/items/e312ddcf9a3d0ea64d72\n",
    "# https://scrapbox.io/miyamonz/pytorch,_transformers%E3%82%92%E4%BD%BF%E3%81%A3%E3%81%9FBERT%E3%81%AEfine-tuning%E3%81%AE%E6%96%B9%E6%B3%95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Z7CSuLLziYn-"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}