example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Differentiable Neural Computer\n",
    "\n",
    "Create synthetic input data `X` of dimension *NxM* where the first *N/2* rows consist of ones and zeros and the last *N/2* rows are zeros. The order of the rows are flipped for target `y` (first *N/2* rows are zeros now). The *DNC* needs to keep this in memory and predict `y` correctly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from model import DNC\n",
    "from trainer import trainer\n",
    "\n",
    "logger = tf.get_logger()\n",
    "logger.setLevel(logging.ERROR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows, cols = 6, 4\n",
    "ones = np.random.randint(0, cols, size=rows)\n",
    "seq = np.zeros((rows, cols))\n",
    "seq[np.arange(rows), ones] = 1\n",
    "zer = np.zeros((rows, cols))\n",
    "\n",
    "X = np.concatenate((seq, zer), axis=0).astype(np.float32)\n",
    "y = np.concatenate((zer, seq), axis=0).astype(np.float32)\n",
    "\n",
    "for i in range(rows):\n",
    "    assert (X[i, :] == y[rows+i,:]).all()\n",
    "\n",
    "X_train = np.expand_dims(X, axis=0)\n",
    "y_train = np.expand_dims(y, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize and train DNC model\n",
    "\n",
    "Initialize:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dnc = DNC(\n",
    "    output_dim=cols,\n",
    "    memory_shape=(10,4),  # shape of memory matrix\n",
    "    n_read=1              # nb of read heads\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer(\n",
    "    model=dnc,\n",
    "    loss_fn=tf.keras.losses.mse,\n",
    "    X_train=X_train,\n",
    "    y_train=y_train,\n",
    "    epochs=2000,\n",
    "    batch_size=1,\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict on `X`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = dnc(X).numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check if the predictions are almost the same as the ground truth `y`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.testing.assert_almost_equal(y_pred, y, decimal=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction: \n",
      "[[-1.922e-03  1.810e-03 -1.225e-04  1.335e-03]\n",
      " [-2.168e-03  2.258e-04 -1.364e-05  2.740e-03]\n",
      " [ 4.904e-04 -6.639e-04 -1.084e-03 -1.633e-03]\n",
      " [-2.993e-03  1.132e-03 -1.938e-04  4.551e-03]\n",
      " [-7.027e-04 -2.482e-03 -7.492e-04 -1.286e-05]\n",
      " [-9.379e-04 -6.188e-04  2.214e-03 -1.392e-03]\n",
      " [-1.105e-03 -1.609e-03  8.935e-04  9.934e-01]\n",
      " [-1.471e-03  9.989e-01 -2.230e-05  4.435e-03]\n",
      " [-2.691e-03  3.490e-03  9.998e-01  2.988e-03]\n",
      " [ 7.629e-05  9.961e-01 -2.395e-04  5.796e-04]\n",
      " [-4.813e-03  1.002e+00  1.611e-04  1.424e-03]\n",
      " [-5.659e-04  9.966e-01  3.111e-04  1.470e-03]]\n",
      "\n",
      "Ground truth: \n",
      "[[0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 0.]\n",
      " [0. 0. 0. 1.]\n",
      " [0. 1. 0. 0.]\n",
      " [0. 0. 1. 0.]\n",
      " [0. 1. 0. 0.]\n",
      " [0. 1. 0. 0.]\n",
      " [0. 1. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(precision=3)\n",
    "print('Prediction: ')\n",
    "print(y_pred)\n",
    "print('\\nGround truth: ')\n",
    "print(y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cdod] *",
   "language": "python",
   "name": "conda-env-cdod-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}