TensorFlow - Text Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "\n",
      "  TensorFlow's `tf-nightly` package will soon be updated to TensorFlow 2.0.\n",
      "\n",
      "  Please upgrade your code to TensorFlow 2.0:\n",
      "    * https://www.tensorflow.org/beta/guide/migration_guide\n",
      "\n",
      "  Or install the latest stable TensorFlow 1.X release:\n",
      "    * `pip install -U \"tensorflow==1.*\"`\n",
      "\n",
      "  Otherwise your code may be broken by the change.\n",
      "\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "# import required libraries\n",
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt\n",
      "1122304/1115394 [==============================] - 1s 1us/step\n"
     ]
    }
   ],
   "source": [
    "# download the shakespeare dataset\n",
    "path_to_file = tf.keras.utils.get_file('shakespeare.txt', 'https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of text: 1115394 characters\n"
     ]
    }
   ],
   "source": [
    "# read the data\n",
    "text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n",
    "\n",
    "# length of text is the number of characters in it\n",
    "print('Length of text: {} characters'.format(len(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You are all resolved rather to die than to famish?\n",
      "\n",
      "All:\n",
      "Resolved. resolved.\n",
      "\n",
      "First Citizen:\n",
      "First, you know Caius Marcius is chief enemy to the people.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# take a look at first 250 chars\n",
    "print(text[:250])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65 unique characters\n"
     ]
    }
   ],
   "source": [
    "# the unique characters in the file\n",
    "vocab = sorted(set(text))\n",
    "print('{} unique characters'.format(len(vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# process the text with vectors\n",
    "\n",
    "# create a mapping from unqiue characters to indices\n",
    "char2idx = {u:i for i, u in enumerate(vocab)}\n",
    "idx2char = np.array(vocab)\n",
    "\n",
    "text_as_int = np.array([char2idx[c] for c in text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  '\\n':   0,\n",
      "  ' ' :   1,\n",
      "  '!' :   2,\n",
      "  '$' :   3,\n",
      "  '&' :   4,\n",
      "  \"'\" :   5,\n",
      "  ',' :   6,\n",
      "  '-' :   7,\n",
      "  '.' :   8,\n",
      "  '3' :   9,\n",
      "  ':' :  10,\n",
      "  ';' :  11,\n",
      "  '?' :  12,\n",
      "  'A' :  13,\n",
      "  'B' :  14,\n",
      "  'C' :  15,\n",
      "  'D' :  16,\n",
      "  'E' :  17,\n",
      "  'F' :  18,\n",
      "  'G' :  19,\n",
      "  ...\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# now we have an integer representation for each character. Notice that we mapped the character as indexes from 0 to len(unique)\n",
    "print('{')\n",
    "for char, _ in zip(char2idx, range(20)):\n",
    "    print('  {:4s}: {:3d},'.format(repr(char), char2idx[char]))\n",
    "print('  ...\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'First Citizen' ----- characters mapped to int ---- > [18 47 56 57 58  1 15 47 58 47 64 43 52]\n"
     ]
    }
   ],
   "source": [
    "# show how the first 13 characters from the text are mappers to integers\n",
    "print('{} ----- characters mapped to int ---- > {}'.format(repr(text[:13]), text_as_int[:13]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F\n",
      "i\n",
      "r\n",
      "s\n",
      "t\n"
     ]
    }
   ],
   "source": [
    "# the prediction task\n",
    "\n",
    "# given a sequence of characters, what is the most probably next character? \n",
    "\n",
    "# This is the task we are training our model to perform\n",
    "\n",
    "\n",
    "# the max length sequence we want for a single input in characters\n",
    "seq_length = 100\n",
    "examples_per_epoch = len(text)//seq_length\n",
    "\n",
    "# create training examples / targets\n",
    "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
    "\n",
    "for i in char_dataset.take(5):\n",
    "    print(idx2char[i.numpy()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou '\n",
      "'are all resolved rather to die than to famish?\\n\\nAll:\\nResolved. resolved.\\n\\nFirst Citizen:\\nFirst, you k'\n",
      "\"now Caius Marcius is chief enemy to the people.\\n\\nAll:\\nWe know't, we know't.\\n\\nFirst Citizen:\\nLet us ki\"\n",
      "\"ll him, and we'll have corn at our own price.\\nIs't a verdict?\\n\\nAll:\\nNo more talking on't; let it be d\"\n",
      "'one: away, away!\\n\\nSecond Citizen:\\nOne word, good citizens.\\n\\nFirst Citizen:\\nWe are accounted poor citi'\n"
     ]
    }
   ],
   "source": [
    "# batch method lets us easily convert these individual characters to sequences of the desired size\n",
    "sequences = char_dataset.batch(seq_length+1, drop_remainder=True)\n",
    "\n",
    "for item in sequences.take(5):\n",
    "    print(repr(''.join(idx2char[item.numpy()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for each sequence duplicate and shift it to form the input and target text by using the map \n",
    "# method to apply a simple function to each batch\n",
    "\n",
    "def split_input_target(chunk):\n",
    "    input_text = chunk[:-1]\n",
    "    target_text = chunk[1:]\n",
    "    return input_text, target_text\n",
    "\n",
    "dataset = sequences.map(split_input_target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input data:  'First Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou'\n",
      "Target data:  'irst Citizen:\\nBefore we proceed any further, hear me speak.\\n\\nAll:\\nSpeak, speak.\\n\\nFirst Citizen:\\nYou '\n"
     ]
    }
   ],
   "source": [
    "# print the first examples input and target values\n",
    "for input_example, target_example in dataset.take(1):\n",
    "    print('Input data: ', repr(''.join(idx2char[input_example.numpy()])))\n",
    "    print('Target data: ', repr(''.join(idx2char[target_example.numpy()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step    0\n",
      "  input: 18 ('F')\n",
      "  expected output: 47 ('i')\n",
      "Step    1\n",
      "  input: 47 ('i')\n",
      "  expected output: 56 ('r')\n",
      "Step    2\n",
      "  input: 56 ('r')\n",
      "  expected output: 57 ('s')\n",
      "Step    3\n",
      "  input: 57 ('s')\n",
      "  expected output: 58 ('t')\n",
      "Step    4\n",
      "  input: 58 ('t')\n",
      "  expected output: 1 (' ')\n"
     ]
    }
   ],
   "source": [
    "# each index of these vectors are processed as one time step. \n",
    "# For the input at time step 0, the model receives the index for \"F\" and tries to predict the index for \"i\" as the next character.\n",
    "# At the next timestep, it does the same thing but RNN considers the previous step context in addition to the current input character\n",
    "\n",
    "for i, (input_idx, target_idx) in enumerate(zip(input_example[:5], target_example[:5])):\n",
    "    print(\"Step {:4d}\".format(i))\n",
    "    print(\"  input: {} ({:s})\".format(input_idx, repr(idx2char[input_idx])))\n",
    "    print(\"  expected output: {} ({:s})\".format(target_idx, repr(idx2char[target_idx])))      \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<DatasetV1Adapter shapes: ((64, 100), (64, 100)), types: (tf.int32, tf.int32)>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create training batches\n",
    "\n",
    "# we use tf.data to split the etxt into manageable sequences. \n",
    "# But before feeding this data into the model, we need to shuffle the data and pack it into batches\n",
    "\n",
    "# batch size\n",
    "BATCH_SIZE = 64\n",
    "steps_per_epoch = examples_per_epoch//BATCH_SIZE\n",
    "\n",
    "# Buffer size to shuffle the dataset\n",
    "# (TF data is designed to work with possible infinite sequences\n",
    "# so it doesn't attempt to shuffle the sequence in memory. Instead,\n",
    "# it maintains a buffer in which it shuffles elements)\n",
    "\n",
    "BUFFER_SIZE = 10000\n",
    "\n",
    "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
    "\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the model\n",
    "\n",
    "# length of the vocab in chars\n",
    "vocab_size = len(vocab)\n",
    "\n",
    "# the embedding dimension\n",
    "embedding_dim = 256\n",
    "\n",
    "# number of RNN units\n",
    "rnn_units = 1024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if tf.test.is_gpu_available():\n",
    "    rnn = tf.keras.layers.CuDNNGRU\n",
    "else:\n",
    "    import functools\n",
    "    rnn = functools.partial(\n",
    "        tf.keras.layers.GRU, recurrent_activation='sigmoid'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "    model = tf.keras.Sequential([\n",
    "        tf.keras.layers.Embedding(\n",
    "            vocab_size,\n",
    "            embedding_dim,\n",
    "            batch_input_shape=[batch_size, None]\n",
    "        ),\n",
    "        rnn(\n",
    "            rnn_units,\n",
    "            return_sequences=True,\n",
    "            recurrent_initializer='glorot_uniform',\n",
    "            stateful=True\n",
    "        ),\n",
    "        tf.keras.layers.Dense(vocab_size)\n",
    "    ])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model(\n",
    "    vocab_size= len(vocab),\n",
    "    embedding_dim= embedding_dim,\n",
    "    rnn_units= rnn_units,\n",
    "    batch_size=BATCH_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 100, 65) # (batch_size, sequence_length, vocab_size)\n"
     ]
    }
   ],
   "source": [
    "# run the model\n",
    "for input_example_batch, target_example_batch in dataset.take(1):\n",
    "    example_batch_predictions = model(input_example_batch)\n",
    "    print(example_batch_predictions.shape, \"# (batch_size, sequence_length, vocab_size)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (64, None, 256)           16640     \n",
      "_________________________________________________________________\n",
      "gru (GRU)                    (64, None, 1024)          3935232   \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (64, None, 65)            66625     \n",
      "=================================================================\n",
      "Total params: 4,018,497\n",
      "Trainable params: 4,018,497\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method _EagerTensorBase.numpy of <tf.Tensor: id=7607, shape=(100,), dtype=int64, numpy=\n",
       "array([ 2, 13, 16, 20, 39, 40,  7, 26, 59, 58,  8,  9,  4, 33, 51, 44, 44,\n",
       "       21, 52,  3,  1, 35, 57, 25, 27, 27, 31, 36, 42, 42, 51, 53, 62, 25,\n",
       "       57, 11,  6,  3, 49, 36, 46, 49, 21, 17,  9, 22, 22, 53, 62, 47,  0,\n",
       "       64, 21, 16, 30, 61, 45, 45,  4, 15, 50, 10,  1, 31, 26, 19, 63, 42,\n",
       "       57, 35, 52, 50, 26, 26, 25,  9,  3, 57,  8, 12,  5, 35, 53,  9, 61,\n",
       "       44, 55, 19, 22, 16, 21, 43, 60, 52,  8, 37, 29, 26, 30, 54],\n",
       "      dtype=int64)>>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now to try for the first example in the batch\n",
    "sampled_indices = tf.random.categorical(\n",
    "    example_batch_predictions[0],\n",
    "    num_samples=1\n",
    ")\n",
    "\n",
    "sampled_indices = tf.squeeze(\n",
    "    sampled_indices,\n",
    "    axis=-1\n",
    ").numpy\n",
    "\n",
    "sampled_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: \n",
      " 'th as this? No, you are deceived;\\ntherefore, back to Rome, and prepare for your\\nexecution: you are c'\n",
      "\n",
      "Next Char Predictions: \n",
      " \"!ADHab-Nut.3&UmffIn$ WsMOOSXddmoxMs;,$kXhkIE3JJoxi\\nzIDRwgg&Cl: SNGydsWnlNNM3$s.?'Wo3wfqGJDIevn.YQNRp\"\n"
     ]
    }
   ],
   "source": [
    "# decode the above to see the text predicted by this untrained model\n",
    "print('Input: \\n', repr(\"\".join(idx2char[input_example_batch[0]])))\n",
    "print()\n",
    "print('Next Char Predictions: \\n', repr(\"\".join(idx2char[sampled_indices()])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction shape:  (64, 100, 65)  # (batch_size, sequence_length, vocab_size)\n",
      "scalar loss:       4.1747603\n"
     ]
    }
   ],
   "source": [
    "# train the model\n",
    "\n",
    "# attach an optimiser and loss function\n",
    "\n",
    "def loss(labels, logits):\n",
    "    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)\n",
    "\n",
    "example_batch_loss = loss(target_example_batch, example_batch_predictions)\n",
    "print(\"Prediction shape: \", example_batch_predictions.shape, \" # (batch_size, sequence_length, vocab_size)\")\n",
    "print(\"scalar loss:      \", example_batch_loss.numpy().mean())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    optimizer= tf.train.AdamOptimizer(),\n",
    "    loss = loss\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# configure checkpoints\n",
    "\n",
    "# directory where the checkpoints will be saved\n",
    "checkpoint_dir = './training_checkpoints'\n",
    "\n",
    "# name of the checkpoint files\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt_{epoch}')\n",
    "\n",
    "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath = checkpoint_prefix,\n",
    "    save_weights_only=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "174/174 [==============================] - 1090s 6s/step - loss: 2.7118\n",
      "Epoch 2/3\n",
      "174/174 [==============================] - 1100s 6s/step - loss: 1.9411\n",
      "Epoch 3/3\n",
      "174/174 [==============================] - 1180s 7s/step - loss: 1.6817\n"
     ]
    }
   ],
   "source": [
    "# execute the training\n",
    "\n",
    "# to keep training time reasonable epochs is set to 3\n",
    "EPOCHS = 3\n",
    "\n",
    "history = model.fit(\n",
    "    dataset.repeat(),\n",
    "    epochs=EPOCHS,\n",
    "    steps_per_epoch=steps_per_epoch,\n",
    "    callbacks=[checkpoint_callback]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./training_checkpoints\\\\ckpt_3'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# generate text\n",
    "\n",
    "# restore the latest checkpoint\n",
    "\n",
    "# to keep this prediction step simple, use a batch size of 1\n",
    "# because of how RNN stateis passed from timestep, the model only accepts a fixed batch size once built\n",
    "# to run the model with a different batch_size, we need to rebuild the model and restore the weights from the\n",
    "# checkpoint\n",
    "\n",
    "tf.train.latest_checkpoint(checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the weights from the checkpoint\n",
    "model = build_model(\n",
    "    vocab_size,\n",
    "    embedding_dim,\n",
    "    rnn_units,\n",
    "    batch_size=1\n",
    ")\n",
    "\n",
    "model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\n",
    "\n",
    "model.build(tf.TensorShape([1,None]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (1, None, 256)            16640     \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (1, None, 1024)           3935232   \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (1, None, 65)             66625     \n",
      "=================================================================\n",
      "Total params: 4,018,497\n",
      "Trainable params: 4,018,497\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# show the summary\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the prediction loop\n",
    "\n",
    "# starts by choosing a start string, initializing the RNN state and setting the number of characters to generate\n",
    "\n",
    "# get the prediction distribution of the next character using the start string and the RNN state\n",
    "\n",
    "# use a multinomial distribution to calculate the index of the predicted character. Use this predicted character\n",
    "# as our next input to the model\n",
    "\n",
    "# the RNN state returned by the model is fed back into the model so that it now has more context, instead of only one word.\n",
    "# After predicting the next word, the modified RNN states are again fed back into the model. Which is how it learns as it \n",
    "# gets more context from the previously predicted words\n",
    "\n",
    "def generate_text(model, start_string):\n",
    "    \n",
    "    # evaluation step (generating text using learned model)\n",
    "    \n",
    "    # Number of characters to generate\n",
    "    num_generate = 1000\n",
    "    \n",
    "    # Converting our starting string to numbers (vectorizing)\n",
    "    input_eval = [char2idx[s] for s in start_string]\n",
    "    input_eval = tf.expand_dims(input_eval, 0)\n",
    "    \n",
    "    # Empty string to store our results\n",
    "    text_generated = []\n",
    "    \n",
    "    # Low temperature results in more predictable text.\n",
    "    # Higher temperature results in more surprising text.\n",
    "    # Experiment to find the best setting\n",
    "    temperature = 1.\n",
    "    \n",
    "    # Here batch size == 1\n",
    "    model.reset_states()\n",
    "    for i in range(num_generate):\n",
    "        predictions = model(input_eval)\n",
    "        \n",
    "        # remove the batch dimension\n",
    "        predictions = tf.squeeze(predictions, 0)\n",
    "        \n",
    "        # using a multinomial distribution to predict the word returned by the model\n",
    "        predictions = predictions / temperature\n",
    "        predicted_id = tf.multinomial(predictions, num_samples=1)[-1,0].numpy()\n",
    "        \n",
    "        # We pass the predicted word as the next input to the model\n",
    "        # along with the previous hidden state\n",
    "        input_eval = tf.expand_dims([predicted_id], 0)\n",
    "        \n",
    "        text_generated.append(idx2char[predicted_id])\n",
    "        \n",
    "    return (start_string + ''.join(text_generated))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tom thou shall parnch\n",
      "I will I suppoth, best wede-in assears.\n",
      "\n",
      "BUCKINGHAM:\n",
      "\n",
      "GLOUCESTER:\n",
      "\n",
      "CORIOLANUS:\n",
      "to shall oppecond thee begantw's distake, your feath. O, what's\n",
      "he is not is yourgell me and exerpse,\n",
      "Then ke propship their ted you daughter.\n",
      "Fwry provise.\n",
      "\n",
      "JIHANTE:\n",
      "I the desirmss of oun if, I spay him huph,\n",
      "And for if that sid, why dower will be sunting cewelthip,\n",
      "And bound know live! I'll nat up a condrieve\n",
      "E culse thy faber word, what I wnamble will live.\n",
      "\n",
      "BINCOFFOLNCI:\n",
      "With our peace fuch were perife:\n",
      "Then that I shall bod\n",
      "Thy frie distignines if in ness now,\n",
      "I dase you gun to beagle.\n",
      "\n",
      "IO:\n",
      "And filly bray in the noth with orrelss, the not\n",
      "Mate no love of our swrine; by nor shoming thy\n",
      "Orfuring; a douch'd\n",
      "And fellows siir; do note, betome, dequesing his Tapliear,\n",
      "The sun to beeath came your bey provess; I heart my feed,\n",
      "Thought upsen the turtie, the oftrown, thou, Great My with wom, downing you; from the gardan meant,\n",
      "Whather he world gove here for his wed.\n",
      "You what I am our prisces.\n",
      "\n",
      "HENRY\n"
     ]
    }
   ],
   "source": [
    "print(generate_text(model, start_string=u\"Tom thou\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Customised Training\n",
    "\n",
    "# the above procedure is simple but does not give you much control\n",
    "\n",
    "# we have seen how to runt he model manually, let's unpack the training loop and implement it ourselves.\n",
    "# This gives a starting point, for example, to implement the curriculum learning to help stabilize the model's open-loop output.\n",
    "\n",
    "# We will use tf.GradientTape to track the gradients. You can learn more about this approach by reading the eager execution guide.\n",
    "\n",
    "# The procedure works as follows:\n",
    "#\n",
    "# First, initalize the RNN state. We do this by calling the tf.keras.Model.reset_states\n",
    "#\n",
    "# Next, iterate over the dataset (batch by batch) and calculate the predictions associated with each.\n",
    "#\n",
    "# Open a tf.GradientTape, and calculate the predictions and loss in that context.\n",
    "#\n",
    "# Calculate the gradients of the loss with respect to the model variables using the tf.GradientTape.grads method\n",
    "#\n",
    "# Finally, take a step downwards by using the optimizer's tf.train.Optimizer.apply_gradients method\n",
    "\n",
    "model = build_model(\n",
    "    vocab_size = len(vocab),\n",
    "    embedding_dim = embedding_dim,\n",
    "    rnn_units = rnn_units,\n",
    "    batch_size=BATCH_SIZE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimiser\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 4.1739\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.global_step\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer.beta1_power\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer.beta2_power\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'm' for (root).layer_with_weights-0.embeddings\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'm' for (root).layer_with_weights-1.cell.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'm' for (root).layer_with_weights-1.cell.recurrent_kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'm' for (root).layer_with_weights-1.cell.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'v' for (root).layer_with_weights-0.embeddings\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'v' for (root).layer_with_weights-1.cell.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'v' for (root).layer_with_weights-1.cell.recurrent_kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.optimizer's state 'v' for (root).layer_with_weights-1.cell.bias\n",
      "WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/alpha/guide/checkpoints#loading_mechanics for details.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-63-1d16af2ced69>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m             \u001b[1;31m# feeding the hidden state back into the model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m             \u001b[1;31m# This is the interesting step\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m             \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m             \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlosses\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msparse_softmax_cross_entropy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredictions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[0;32m    896\u001b[0m           with base_layer_utils.autocast_context_manager(\n\u001b[0;32m    897\u001b[0m               self._compute_dtype):\n\u001b[1;32m--> 898\u001b[1;33m             \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcast_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    899\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle_activity_regularization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    900\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_mask_metadata\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_masks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\sequential.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, inputs, training, mask)\u001b[0m\n\u001b[0;32m    253\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    254\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_init_graph_network\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 255\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSequential\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    256\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    257\u001b[0m     \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minputs\u001b[0m  \u001b[1;31m# handle the corner case where self.layers is empty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\network.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, inputs, training, mask)\u001b[0m\n\u001b[0;32m    693\u001b[0m                                 ' implement a `call` method.')\n\u001b[0;32m    694\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 695\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_internal_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    696\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    697\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0mcompute_output_shape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\network.py\u001b[0m in \u001b[0;36m_run_internal_graph\u001b[1;34m(self, inputs, training, mask)\u001b[0m\n\u001b[0;32m    842\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    843\u001b[0m           \u001b[1;31m# Compute outputs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 844\u001b[1;33m           \u001b[0moutput_tensors\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcomputed_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    845\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    846\u001b[0m           \u001b[1;31m# Update tensor_dict.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs, initial_state, constants, **kwargs)\u001b[0m\n\u001b[0;32m    621\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    622\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0minitial_state\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mconstants\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 623\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mRNN\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    624\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    625\u001b[0m     \u001b[1;31m# If any of `initial_state` or `constants` are specified and are Keras\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs, *args, **kwargs)\u001b[0m\n\u001b[0;32m    896\u001b[0m           with base_layer_utils.autocast_context_manager(\n\u001b[0;32m    897\u001b[0m               self._compute_dtype):\n\u001b[1;32m--> 898\u001b[1;33m             \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcast_inputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    899\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_handle_activity_regularization\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    900\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_mask_metadata\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput_masks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, inputs, mask, training, initial_state)\u001b[0m\n\u001b[0;32m   1920\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_recurrent_dropout_mask\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1921\u001b[0m     return super(GRU, self).call(\n\u001b[1;32m-> 1922\u001b[1;33m         inputs, mask=mask, training=training, initial_state=initial_state)\n\u001b[0m\u001b[0;32m   1923\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1924\u001b[0m   \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, inputs, mask, training, initial_state, constants)\u001b[0m\n\u001b[0;32m    754\u001b[0m         \u001b[0minput_length\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minput_length\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    755\u001b[0m         \u001b[0mtime_major\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime_major\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 756\u001b[1;33m         zero_output_for_mask=self.zero_output_for_mask)\n\u001b[0m\u001b[0;32m    757\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstateful\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    758\u001b[0m       \u001b[0mupdates\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\backend.py\u001b[0m in \u001b[0;36mrnn\u001b[1;34m(step_function, inputs, initial_states, go_backwards, mask, constants, unroll, input_length, time_major, zero_output_for_mask)\u001b[0m\n\u001b[0;32m   4050\u001b[0m           \u001b[0mbody\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_step\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4051\u001b[0m           \u001b[0mloop_vars\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutput_ta\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4052\u001b[1;33m           **while_loop_kwargs)\n\u001b[0m\u001b[0;32m   4053\u001b[0m       \u001b[0mnew_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfinal_outputs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4054\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\ops\\control_flow_ops.py\u001b[0m in \u001b[0;36mwhile_loop\u001b[1;34m(cond, body, loop_vars, shape_invariants, parallel_iterations, back_prop, swap_memory, name, maximum_iterations, return_same_structure)\u001b[0m\n\u001b[0;32m   2712\u001b[0m                                               list(loop_vars))\n\u001b[0;32m   2713\u001b[0m       \u001b[1;32mwhile\u001b[0m \u001b[0mcond\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mloop_vars\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2714\u001b[1;33m         \u001b[0mloop_vars\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mloop_vars\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2715\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtry_to_pack\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloop_vars\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_basetuple\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2716\u001b[0m           \u001b[0mpacked\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\backend.py\u001b[0m in \u001b[0;36m_step\u001b[1;34m(time, output_ta_t, *states)\u001b[0m\n\u001b[0;32m   4034\u001b[0m         \u001b[0mcurrent_input\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpack_sequence_as\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcurrent_input\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4035\u001b[0m         output, new_states = step_function(current_input,\n\u001b[1;32m-> 4036\u001b[1;33m                                            tuple(states) + tuple(constants))\n\u001b[0m\u001b[0;32m   4037\u001b[0m         \u001b[0mflat_state\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4038\u001b[0m         \u001b[0mflat_new_state\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(inputs, states)\u001b[0m\n\u001b[0;32m    730\u001b[0m       \u001b[1;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    731\u001b[0m         \u001b[0mstates\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstates\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mis_tf_rnn_cell\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 732\u001b[1;33m         \u001b[0moutput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcell\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    733\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mnest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_sequence\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_states\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    734\u001b[0m           \u001b[0mnew_states\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mnew_states\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, inputs, states, training)\u001b[0m\n\u001b[0;32m   1669\u001b[0m       \u001b[0mrecurrent_z\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mh_tm1_z\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecurrent_kernel\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munits\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1670\u001b[0m       recurrent_r = K.dot(h_tm1_r,\n\u001b[1;32m-> 1671\u001b[1;33m                           self.recurrent_kernel[:, self.units:self.units * 2])\n\u001b[0m\u001b[0;32m   1672\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_after\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muse_bias\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1673\u001b[0m         \u001b[0mrecurrent_z\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbias_add\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrecurrent_z\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrecurrent_bias\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munits\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\ops\\array_ops.py\u001b[0m in \u001b[0;36m_SliceHelperVar\u001b[1;34m(var, slice_spec)\u001b[0m\n\u001b[0;32m   1040\u001b[0m   \"\"\"\n\u001b[0;32m   1041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1042\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0m_slice_helper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvar\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mslice_spec\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvar\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1043\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1044\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\ops\\array_ops.py\u001b[0m in \u001b[0;36m_slice_helper\u001b[1;34m(tensor, slice_spec, var)\u001b[0m\n\u001b[0;32m    800\u001b[0m         \u001b[0mellipsis_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mellipsis_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    801\u001b[0m         \u001b[0mvar\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvar\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 802\u001b[1;33m         name=name)\n\u001b[0m\u001b[0;32m    803\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    804\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\ops\\array_ops.py\u001b[0m in \u001b[0;36mstrided_slice\u001b[1;34m(input_, begin, end, strides, begin_mask, end_mask, ellipsis_mask, new_axis_mask, shrink_axis_mask, var, name)\u001b[0m\n\u001b[0;32m    966\u001b[0m       \u001b[0mellipsis_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mellipsis_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    967\u001b[0m       \u001b[0mnew_axis_mask\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnew_axis_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 968\u001b[1;33m       shrink_axis_mask=shrink_axis_mask)\n\u001b[0m\u001b[0;32m    969\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    970\u001b[0m   \u001b[0mparent_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Continuum\\anaconda3\\envs\\py36\\lib\\site-packages\\tensorflow_core\\python\\ops\\gen_array_ops.py\u001b[0m in \u001b[0;36mstrided_slice\u001b[1;34m(input, begin, end, strides, begin_mask, end_mask, ellipsis_mask, new_axis_mask, shrink_axis_mask, name)\u001b[0m\n\u001b[0;32m  10351\u001b[0m         \u001b[0mend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstrides\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"begin_mask\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbegin_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"end_mask\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m  10352\u001b[0m         \u001b[1;34m\"ellipsis_mask\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mellipsis_mask\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"new_axis_mask\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_axis_mask\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m> 10353\u001b[1;33m         \"shrink_axis_mask\", shrink_axis_mask)\n\u001b[0m\u001b[0;32m  10354\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m  10355\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_FallbackException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# training step\n",
    "EPOCHS = 1\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    start = time.time()\n",
    "    \n",
    "    # initializing the hidden state at the start of every epoch\n",
    "    # initially hidden is None\n",
    "    hidden = model.reset_states()\n",
    "    \n",
    "    for (batch_n, (inp, target)) in enumerate(dataset):\n",
    "        with tf.GradientTape() as tape:\n",
    "            \n",
    "            # feeding the hidden state back into the model\n",
    "            # This is the interesting step\n",
    "            predictions = model(inp)\n",
    "            loss = tf.losses.sparse_softmax_cross_entropy(target, predictions)\n",
    "            \n",
    "        grads = tape.gradient(loss, model.trainable_variables)\n",
    "        optimizer.apply_gradients(zip(grads, model.trainable_variables))\n",
    "        \n",
    "        if batch_n % 100 == 0:\n",
    "            template = 'Epoch {} Batch {} Loss {:.4f}'\n",
    "            print(template.format(epoch+1, batch_n, loss))\n",
    "            \n",
    "    # saving (checkpoint) the model every 5 epochs\n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        model.save_weights(checkpoint_prefix.format(epoch=epoch))\n",
    "        \n",
    "    print('Epoch {} Loss {:.4f}'.format(time.time() - start))\n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n",
    "    \n",
    "model.save_weights(checkpoint_prefix.format(epoch=epoch))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}