seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!curl -o data/fra-eng.zip http://www.manythings.org/anki/fra-eng.zip\n",
    "#!unzip data/fra-eng.zip -d data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading Data and Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<S>', 'stay', 'thin', '.', '</S>']\n",
      "['<S>', 'reste', 'mince', '!', '</S>']\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "data = pd.read_csv(\"data/fra.txt\",sep='\\t',header= None,encoding=\"utf-8\")\n",
    "ang = list(data[0])\n",
    "fra = list(data[1])\n",
    "\n",
    "from spacy.lang.en import English\n",
    "from spacy.lang.fr import French\n",
    "\n",
    "nlp = English()\n",
    "tokenizer = nlp.Defaults.create_tokenizer(nlp)\n",
    "nlp_f = French()\n",
    "tokenizer_f = nlp_f.Defaults.create_tokenizer(nlp_f)\n",
    "\n",
    "ang = [['<S>'] + [token.string.strip() for token in tokenizer(text.lower())] + ['</S>'] for text in ang][:10000]\n",
    "\n",
    "fra = [['<S>'] + [token.string.strip() for token in tokenizer_f(text.lower())] + ['</S>'] for text in fra][:10000]\n",
    "\n",
    "print(ang[1000])\n",
    "print(fra[1000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "import numpy as np\n",
    "\n",
    "EMBEDDING_SIZE = 120\n",
    "w2v = Word2Vec(ang, size=EMBEDDING_SIZE, window=10, min_count=1, negative=10, workers=10)\n",
    "word_map = {}\n",
    "word_map[\"<PAD>\"] = 0\n",
    "word_vectors = [np.zeros((EMBEDDING_SIZE,))]\n",
    "for i, w in enumerate([w for w in w2v.wv.vocab]):\n",
    "    word_map[w] = i+1\n",
    "    word_vectors.append(w2v.wv[w])\n",
    "word_vectors = np.vstack(word_vectors)\n",
    "\n",
    "w2v = Word2Vec(fra, size=EMBEDDING_SIZE, window=10, min_count=1, negative=10, workers=10)\n",
    "word_map_fr = {}\n",
    "word_map_fr[\"<PAD>\"] = 0\n",
    "word_vectors_fr = [np.zeros((EMBEDDING_SIZE,))]\n",
    "for i, w in enumerate([w for w in w2v.wv.vocab]):\n",
    "    word_map_fr[w] = i+1\n",
    "    word_vectors_fr.append(w2v.wv[w])\n",
    "word_vectors_fr = np.vstack(word_vectors_fr)\n",
    "i2w = dict(zip([*word_map_fr.values()],[*word_map_fr]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Padding, word2id and shifting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad(a,shift = False):\n",
    "    shape = len(a)\n",
    "    max_s = max([len(x) for x in a])\n",
    "    token = np.zeros((shape,max_s+1),dtype = np.int)\n",
    "    mask  =  np.zeros((shape,max_s+1),dtype = np.int)\n",
    "    for i,o in enumerate(a):\n",
    "        token[i,:len(o)] = o\n",
    "        mask[i,:len(o)] = 1\n",
    "    if shift:\n",
    "        return token[:,:-1],token[:,1:],mask[:,1:],max_s\n",
    "    else:\n",
    "        return token[:,:-1],mask[:,:-1],max_s\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ang_tok,ang_mask,ang_pl = pad([[word_map[w] for w in text] for text in ang])\n",
    "fra_tok,fra_tok_t,fra_mask,fr_pl = pad([[word_map_fr[w] for w in text] for text in fra],shift = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Seq2Seq model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import layers,Model\n",
    "from tensorflow.keras.initializers import Constant\n",
    "import tensorflow as tf\n",
    "\n",
    "class S2S(tf.keras.Model):\n",
    "    def __init__(self,Win,Wout,i2w):\n",
    "      \n",
    "        super(S2S, self).__init__() \n",
    "        \n",
    "        self.nv_in = Win.shape[0]\n",
    "        self.r = Win.shape[1]\n",
    "        self.nv_out = Wout.shape[0]\n",
    "        \n",
    "        self.i2w = i2w\n",
    "        \n",
    "        self.Win = layers.Embedding(self.nv_in,self.r)\n",
    "        self.Win.build((None, ))\n",
    "        self.Win.set_weights([Win])\n",
    "        self.Win.trainable = True\n",
    "\n",
    "        self.Wout = layers.Embedding(self.nv_out,self.r)\n",
    "        self.Wout.build((None, ))\n",
    "        self.Wout.set_weights([Wout])\n",
    "        self.Wout.trainable = True\n",
    "        \n",
    "        self.encoder = layers.GRU(self.r,dropout=0.2)\n",
    "        \n",
    "        self.decoder = layers.GRU(self.r, return_sequences=True, return_state=True,dropout=0.2)\n",
    "        \n",
    "        self.mapper = layers.Dense(self.nv_out,activation = \"softmax\")\n",
    "\n",
    "    @tf.function\n",
    "    def call(self,x,x_mask,x_out):\n",
    "        x = self.Win(x)\n",
    "        x_mask = tf.cast(x_mask,dtype=bool)\n",
    "    \n",
    "        x = self.encoder(x,mask=x_mask)   \n",
    "        \n",
    "        x_out = self.Wout(x_out)\n",
    "        out,_ = self.decoder(x_out, initial_state=x)\n",
    "        \n",
    "        probs = self.mapper(out)\n",
    "\n",
    "        return probs\n",
    "\n",
    "    def generate(self,start_emb,stop_emb,x,x_mask):\n",
    "        aout = []\n",
    "        \n",
    "        x = tf.expand_dims(x,axis=0)\n",
    "        x_mask = tf.expand_dims(x_mask,axis=0)\n",
    "\n",
    "        x_mask = tf.cast(x_mask,dtype=bool)\n",
    "        \n",
    "        x = self.Win(x)\n",
    "\n",
    "        x = self.encoder(x,mask=x_mask) \n",
    "        \n",
    "        x_out = tf.expand_dims(tf.expand_dims(self.Wout(start_emb),axis = 0),axis = 0)         \n",
    "        _,out = self.decoder(x_out, initial_state=x) \n",
    "        probs = tf.squeeze(self.mapper(out))\n",
    "        \n",
    "        x_out = tf.math.argmax(probs)\n",
    "        val,argval = tf.nn.top_k(probs, k=2, sorted=True, name=None)\n",
    "        x_out = argval.numpy()[0]\n",
    "        aout.append(self.i2w[x_out])\n",
    "        \n",
    "        for i in range(10):\n",
    "            x_out = tf.expand_dims(tf.expand_dims(self.Wout(tf.constant(x_out)),axis = 0),axis = 0)     \n",
    "            _,out = self.decoder(x_out, initial_state=out)  \n",
    "            \n",
    "            probs = tf.squeeze(self.mapper(out))\n",
    "            val,argval = tf.nn.top_k(probs, k=2, sorted=True, name=None)\n",
    "            x_out = argval.numpy()[0]\n",
    "            aout.append(self.i2w[x_out])\n",
    "            \n",
    "            if x_out == stop_emb:\n",
    "                break\n",
    "            \n",
    "        return aout\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function for optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def compute_loss(model,loss_f,x,x_mask,x_out,y_out,y_out_mask):\n",
    "    \n",
    "    \n",
    "    probs = model(x,x_mask,x_out)\n",
    "    \n",
    "    y_true= tf.boolean_mask(y_out,y_out_mask)\n",
    "    y_pred = tf.boolean_mask(probs,y_out_mask) \n",
    "    \n",
    "    return loss_f(y_true,y_pred),y_true,y_pred\n",
    "\n",
    "\n",
    "@tf.function\n",
    "def compute_apply_gradients(model,loss_f,x,x_mask,x_out,y_out,y_out_mask, optimizer):\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        \n",
    "        loss,label,prediction= compute_loss(model, loss_f,x,x_mask,x_out,y_out,y_out_mask)\n",
    "        \n",
    "    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "    \n",
    "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "    \n",
    "    return loss,label,prediction\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 3\n",
    "\n",
    "X = np.hstack([ang_tok,ang_mask,fra_tok])\n",
    "print(X.shape)\n",
    "Y = np.hstack([fra_tok_t,fra_mask])\n",
    "print(Y.shape)\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, train_size=0.80, random_state=101)\n",
    "\n",
    "\n",
    "train_data = tf.data.Dataset.from_tensor_slices((X_train,Y_train)).batch(batch_size)\n",
    "\n",
    "test_data = tf.data.Dataset.from_tensor_slices((X_test,Y_test)).batch(batch_size)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model declaration and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = S2S(word_vectors,word_vectors_fr,i2w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "\n",
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.CategoricalAccuracy(name='train_accuracy')\n",
    "\n",
    "test_loss = tf.keras.metrics.Mean(name='test_loss')\n",
    "test_accuracy = tf.keras.metrics.CategoricalAccuracy(name='test_accuracy')\n",
    "\n",
    "loss_f = tf.keras.losses.CategoricalCrossentropy()\n",
    "    \n",
    "optimizer = tf.keras.optimizers.RMSprop(learning_rate=0.0001)\n",
    "loss_f = tf.keras.losses.CategoricalCrossentropy()\n",
    "epochs = 30\n",
    "\n",
    "checkpoint_dir = 'training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 model=model)\n",
    "\n",
    "tr_loss = []\n",
    "te_loss = []\n",
    "tr_acc = []\n",
    "te_acc = []\n",
    "    \n",
    "for epoch in range(1, epochs + 1):\n",
    "    print(epoch,flush=True,)\n",
    "\n",
    "    for x,y in tqdm(train_data):\n",
    "        \n",
    "        x,x_mask,x_out = tf.split(x,[ang_pl,ang_pl,fr_pl],axis=1)\n",
    "        y_out,y_out_mask = tf.split(y,2,axis=1)\n",
    "        \n",
    "        y_out_onehot = tf.one_hot(y_out,depth = word_vectors_fr.shape[0])\n",
    "        \n",
    "        loss,label,prediction = compute_apply_gradients(model,loss_f,x,x_mask,x_out,y_out_onehot,y_out_mask,optimizer)\n",
    "\n",
    "        train_loss(loss)\n",
    "        train_accuracy(label, prediction)\n",
    "        \n",
    "    for x,y in tqdm(test_data):\n",
    "\n",
    "        x,x_mask,x_out = tf.split(x,[ang_pl,ang_pl,fr_pl],axis=1)\n",
    "        y_out,y_out_mask = tf.split(y,2,axis=1)\n",
    "\n",
    "        y_out_onehot = tf.one_hot(y_out,depth = word_vectors_fr.shape[0])\n",
    "\n",
    "        loss,label,prediction = compute_loss(model,loss_f,x,x_mask,x_out,y_out_onehot,y_out_mask)\n",
    "        test_loss(loss)\n",
    "        test_accuracy(label, prediction)\n",
    "                \n",
    "    print(\" \".join(ang[1000]))\n",
    "    print(\" \".join(model.generate(word_map_fr[\"<S>\"],word_map_fr[\"</S>\"],ang_tok[1000,:],ang_mask[1000,:])))\n",
    "    \n",
    "    print(\" \".join(ang[5000]))\n",
    "    print(\" \".join(model.generate(word_map_fr[\"<S>\"],word_map_fr[\"</S>\"],ang_tok[5000,:],ang_mask[5000,:])))\n",
    "    \n",
    "    print(\n",
    "    f'Loss: {train_loss.result()}, '\n",
    "    f'Accuracy: {train_accuracy.result() * 100}, '\n",
    "    f'Test Loss: {test_loss.result()}, '\n",
    "    f'Test Accuracy: {test_accuracy.result() * 100}')\n",
    "    \n",
    "    tr_loss.append(train_loss.result())\n",
    "    te_loss.append(test_loss.result())\n",
    "    tr_acc.append(train_accuracy.result())\n",
    "    te_acc.append(test_accuracy.result())\n",
    "    if epoch % 2 == 0:\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.plot(tr_loss, c=\"orange\")\n",
    "plt.plot(te_loss, c=\"blue\")\n",
    "plt.title(\"Loss\")\n",
    "plt.show()\n",
    "\n",
    "plt.clf()\n",
    "plt.plot(tr_acc, c=\"orange\")\n",
    "plt.plot(te_acc, c=\"red\")\n",
    "plt.title(\"Accuracy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}