final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instructions for reproduction -\n",
    "\n",
    "I expect you'd want to run this on a cluster like MAMBA. You'll need two scripts to submit your job -\n",
    "\n",
    "`do.sh` - https://paste.rs/\n",
    "1a4\n",
    "\n",
    "```\n",
    "#!/bin/bash\n",
    "\n",
    "#rm -rf semisupervised_nlu\n",
    "\n",
    "git clone --recurse-submodules https://github.com/DhruvDh/semisupervised_nlu.git\n",
    "\n",
    "job=$(qsub -d `pwd` -l nodes=1:ppn=16:gpus=1 submit.sh -q mamba -l walltime=11:30:00)\n",
    "num=${job:0:5}\n",
    "\n",
    "echo \"Job ID: $num\"\n",
    "echo $num > lastjob\n",
    "```\n",
    "\n",
    "`submit.sh` - https://paste.rs/xxK\n",
    "\n",
    "```\n",
    "#!/bin/bash\n",
    "\n",
    "export PATH=~/.local/bin:$PATH\n",
    "\n",
    "#module load cuda/8.0 cudnn/6.0-cuda8 anaconda3/5.0.1-cuda8\n",
    "\n",
    "module load pytorch/1.2.0-anaconda3-cuda10.0 \n",
    "python3 -m pip install --user transformers tensorboardx\n",
    "\n",
    "source ~/.bashrc\n",
    "conda init\n",
    "source ~/.bashrc\n",
    "\n",
    "cd semisupervised_nlu\n",
    "\n",
    "python3 ./transformers/examples/run_lm_finetuning.py \\\n",
    "    --output_dir=output \\\n",
    "    --model_type=roberta \\\n",
    "    --model_name_or_path=roberta-base \\\n",
    "    --do_train \\\n",
    "    --train_data_file=\"./data/roberta/train.txt\" \\\n",
    "    --do_eval \\\n",
    "    --eval_data_file=\"./data/roberta/test.txt\" \\\n",
    "    --num_train_epochs=15 \\\n",
    "    --save_steps=659 \\\n",
    "    --save_total_limit=2 \\\n",
    "    --mlm\n",
    "```\n",
    "\n",
    "And run `do.sh` to submit the training job. This will produce a model in `semisupervised_nlu/output`, which is where this notebook will try to load the model from later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import RobertaForMaskedLM, RobertaTokenizer\n",
    "import torch\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finetuned performance (only 15 epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from random import choice\n",
    "\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_intents = os.path.join('data', 'raw')\n",
    "intents = os.listdir(path_to_intents)\n",
    "get_path = lambda x: os.path.join('data', 'raw', x, x + \".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\dhruv\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "os.chdir('helpers')\n",
    "from analyze import questions, entities, get_data\n",
    "data = get_data()\n",
    "os.chdir('..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = RobertaForMaskedLM.from_pretrained('./output/')\n",
    "tokenizer = RobertaTokenizer.from_pretrained('./output/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "text = defaultdict(list)\n",
    "encoded_text = defaultdict(list)\n",
    "\n",
    "ans = defaultdict(list)\n",
    "response = defaultdict(list)\n",
    "\n",
    "scores = defaultdict(list)\n",
    "\n",
    "for (_questions, intent, entity) in zip(questions, intents, entities):\n",
    "    question = choice(_questions)\n",
    "    \n",
    "    for (r, e) in zip(data[intent]['df']['text'], data[intent]['df'][entity]):\n",
    "        text[intent].append(\"<s> \" + r.strip() + '. ' + question.strip() + \" \".join([\"<mask>\" for x in str(e).split()]) + \" </s>\")\n",
    "        ans[intent].append((str(e)).split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for intent in intents:\n",
    "    for row in text[intent]:\n",
    "        encoded_text[intent].append(torch.tensor([tokenizer.encode(row, add_special_tokens=False)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2042it [03:13, 10.56it/s]\n",
      "2073it [02:58, 11.59it/s]\n",
      "2100it [03:39,  9.55it/s]\n",
      "2100it [02:40, 13.08it/s]\n",
      "2056it [02:41, 12.72it/s]\n",
      "2054it [02:47, 12.25it/s]\n",
      "2059it [03:01, 11.33it/s]\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    for intent in intents:\n",
    "        for i, row in tqdm(enumerate(encoded_text[intent])):\n",
    "            out = model(row)\n",
    "            response[intent].append(tokenizer.decode(torch.argmax(out[0][0], dim=1).tolist()).split())\n",
    "            \n",
    "            score = nltk.translate.bleu_score.sentence_bleu(\n",
    "                    [ans[intent][i]],\n",
    "                    response[intent][i],\n",
    "                    smoothing_function=nltk.translate.bleu_score.SmoothingFunction().method4,\n",
    "                    auto_reweigh=True\n",
    "                )\n",
    "            scores[intent].append(score)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For AddToPlaylist, avg. BLEU score is 0.160211052589337\n",
      "For BookRestaurant, avg. BLEU score is 0.1656835101344695\n",
      "For GetWeather, avg. BLEU score is 0.0992932125200511\n",
      "For PlayMusic, avg. BLEU score is 0.12332243402123681\n",
      "For RateBook, avg. BLEU score is 0.13669591041649146\n",
      "For SearchCreativeWork, avg. BLEU score is 0.20891380704355164\n",
      "For SearchScreeningEvent, avg. BLEU score is 0.20672132657334064\n"
     ]
    }
   ],
   "source": [
    "for intent in intents:\n",
    "    print(f\"For {intent}, avg. BLEU score is {sum(scores[intent]) / len(scores[intent])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}