rbm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "Torch       :  Manipulating vectors like dot product, addition etc and using GPU\n",
    "Numpy       :  Manipuliting vectors\n",
    "Pandas      :  Reading CSV file\n",
    "Matplotlib  :  Plotting figure\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "            O\n",
    "            O\n",
    "            O\n",
    "            O\n",
    "            O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     O\n",
    "            O                     |\n",
    "            O                     |\n",
    "            O                     |\n",
    "            O                     |\n",
    "            O                     |\n",
    "            |                     |\n",
    "            |                     |\n",
    "            |                     |\n",
    "            |                     |\n",
    "            |                     |\n",
    "            |                     |\n",
    "            |                     |\n",
    "         Visible           Hidden/Feature\n",
    "          Layer                 Layer\n",
    "          (n_v)                 (n_h)\n",
    "          \n",
    "          \n",
    "RBM  :  A class that initialize RBM with default values\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "   Parameters\n",
    "\n",
    "   n_v             :   Number of visible inputs\n",
    "                       Initialized by 0 but then take value of number of inputs\n",
    "   n_h             :   Number of features want to extract\n",
    "                       Must be set by user\n",
    "   k               :   Sampling steps for contrastive divergance\n",
    "                       Default value is 2 steps\n",
    "   epochs          :   Number of epochs for training RBM\n",
    "                       Must be set by user\n",
    "   mini_batch_size :   Size of mini batch for training\n",
    "                       Must be set by user\n",
    "   alpha           :   Learning rate for updating parameters of RBM\n",
    "                       Default value is 0.01\n",
    "   momentum        :   Reduces large jumps for updating parameters\n",
    "   weight_decay    :   Reduces the value of weight after every step of contrastive divergance\n",
    "   data            :   Data to be fitted for RBM\n",
    "                       Must be given by user or else, thats all useless\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RBM():\n",
    "\n",
    "#   Parameters\n",
    "\n",
    "#   n_v             :   Number of visible inputs\n",
    "#                       Initialized by 0 but then take value of number of inputs\n",
    "#   n_h             :   Number of features want to extract\n",
    "#                       Must be set by user\n",
    "#   k               :   Sampling steps for contrastive divergance\n",
    "#                       Default value is 2 steps\n",
    "#   epochs          :   Number of epochs for training RBM\n",
    "#                       Must be set by user\n",
    "#   mini_batch_size :   Size of mini batch for training\n",
    "#                       Must be set by user\n",
    "#   alpha           :   Learning rate for updating parameters of RBM\n",
    "#                       Default value is 0.001\n",
    "#   momentum        :   Reduces large jumps for updating parameters\n",
    "#   weight_decay    :   Reduces the value of weight after every step of contrastive divergance\n",
    "#   data            :   Data to be fitted for RBM\n",
    "#                       Must be given by user or else, thats all useless\n",
    "    \n",
    "    def __init__(self, n_v=0, n_h=0, k=2, epochs=15, mini_batch_size=64, alpha=0.001, momentum=0.9, weight_decay=0.001):\n",
    "        self.number_features = 0\n",
    "        self.n_v             = n_v\n",
    "        self.n_h             = self.number_features\n",
    "        self.k               = k\n",
    "        self.alpha           = alpha\n",
    "        self.momentum        = momentum\n",
    "        self.weight_decay    = weight_decay\n",
    "        self.mini_batch_size = mini_batch_size\n",
    "        self.epochs          = epochs\n",
    "        self.data            = torch.randn(1, device=\"cuda\")\n",
    "\n",
    "#   fit            method is called to fit RBM for provided data\n",
    "#                  First, data is converted in range of 0-1 cuda float tensors by dividing it by their maximum value\n",
    "#                  Here, after calling this method, n_v is reinitialized to number of input values present in data\n",
    "#                  number_features must be given by user before calling this method\n",
    "#   w              Tensor of weights of RBM\n",
    "#                  (n_v x n_h)    Randomly initialized between 0-1\n",
    "#   a              Tensor of bias for visible units\n",
    "#                  (n_v x 1)      Initialized by 1's\n",
    "#   b              Tensor of bias for hidden units\n",
    "#                  (n_b x 1)      Initialized by 1's\n",
    "#   w_moment       Momentum value for weights\n",
    "#                  (n_v x n_h)    Initialized by zeros\n",
    "#   a_moment       Momentum values for visible units\n",
    "#                  (n_v x 1)      Initialized by zeros\n",
    "#   b_moment       Momentum values for hidden units\n",
    "#                  (n_h x 1)      Initialized by zeros\n",
    "    def fit(self):\n",
    "        \n",
    "        self.data /= self.data.max()\n",
    "        \n",
    "        self.data = self.data.type(torch.cuda.FloatTensor)\n",
    "        \n",
    "        self.n_v = len(self.data[0])\n",
    "        self.n_h = self.number_features\n",
    "        \n",
    "        self.w = torch.randn(self.n_v, self.n_h, device=\"cuda\") * 0.1\n",
    "        self.a = torch.ones(self.n_v, device=\"cuda\") * 0.5\n",
    "        self.b = torch.ones(self.n_h, device=\"cuda\")\n",
    "\n",
    "        self.w_moment = torch.zeros(self.n_v, self.n_h, device=\"cuda\")\n",
    "        self.a_moment = torch.zeros(self.n_v, device=\"cuda\")\n",
    "        self.b_moment = torch.zeros(self.n_h, device=\"cuda\")\n",
    "        \n",
    "        self.train()\n",
    "\n",
    "#   train          This method splits dataset into mini_batch and run for given epoch number of times\n",
    "    def train(self):\n",
    "        for epoch_no in range(self.epochs):\n",
    "            ep_error = 0\n",
    "            for i in range(0, len(self.data), self.mini_batch_size):\n",
    "                mini_batch = self.data[i:i+self.mini_batch_size]\n",
    "                ep_error += self.contrastive_divergence(mini_batch)\n",
    "            print(\"Epoch Number  : \", epoch_no, \"       Error  : \", ep_error.item())\n",
    "\n",
    "#   cont_diverg    It performs contrastive divergance using gibbs sampling algorithm\n",
    "#   p_h_0          Value of hidden units for given visivle units\n",
    "#   h_0            Activated hidden units as sampled from normal distribution (0 or 1)\n",
    "#   g_0            Positive associations of RBM\n",
    "#   wv_a           Unactivated hidden units\n",
    "#   p_v_h          Probability of hidden neuron to be activated given values of visible neurons\n",
    "#   p_h_v          Probability of visible neuron to be activated given values of hidden neurons\n",
    "#   p_v_k          Value of visible units for given visivle units after k step Gibbs Sampling\n",
    "#   p_h_k          Value of hidden units for given visivle units after k step Gibbs Sampling\n",
    "#   g_k            Negative associations of RBM\n",
    "#   error          Recontruction error for given mini_batch\n",
    "    def contrastive_divergence(self, v):\n",
    "        p_h_0 = self.sample_hidden(v)\n",
    "        h_0   = (p_h_0 >= torch.rand(self.n_h, device=\"cuda\")).float()\n",
    "        g_0   = v.transpose(0, 1).mm(h_0)\n",
    "\n",
    "        wv_a = h_0\n",
    "#       Gibbs Sampling step\n",
    "        for step in range(self.k):\n",
    "            p_v_h = self.sample_visible(wv_a)\n",
    "            p_h_v = self.sample_hidden(p_v_h)\n",
    "            wv_a  = (p_h_v >= torch.rand(self.n_h, device=\"cuda\")).float()\n",
    "\n",
    "        p_v_k = p_v_h\n",
    "        p_h_k = p_h_v\n",
    "\n",
    "        g_k = p_v_k.transpose(0, 1).mm(p_h_k)\n",
    "\n",
    "        self.update_parameters(g_0, g_k, v, p_v_k, p_h_0, p_h_k)\n",
    "\n",
    "        error = torch.sum((v - p_v_k)**2)\n",
    "\n",
    "        return error\n",
    "\n",
    "#   p_v_h     :   Probability of hidden neuron to be activated given values of visible neurons\n",
    "#   p_h_v     :   Probability of visible neuron to be activated given values of hidden neurons\n",
    "\n",
    "#-----------------------------------Bernoulli-Bernoulli RBM--------------------------------------------\n",
    "#   p_h_v    =    sigmoid ( weight  x visible  +  visible_bias )\n",
    "#   p_v_h    =    sigmoid (weight.t x hidden   +  hidden_bias )\n",
    "#------------------------------------------------------------------------------------------------------\n",
    "    def sample_hidden(self, p_v_h):  #   Bernoulli-Bernoulli RBM\n",
    "        wv    = p_v_h.mm(self.w)\n",
    "        wv_a  = wv + self.b\n",
    "        p_h_v = torch.sigmoid(wv_a)\n",
    "        return p_h_v\n",
    "\n",
    "    def sample_visible(self, p_h_v): #   Bernoulli-Bernoulli RBM\n",
    "        wh    = p_h_v.mm(self.w.transpose(0, 1))\n",
    "        wh_b  = wh + self.a\n",
    "        p_v_h = torch.sigmoid(wh_b)\n",
    "        return p_v_h\n",
    "\n",
    "#   weight_(t)        =     weight_(t)       +  ( positive_association  -  negative_association )  +  weight_(t-1)\n",
    "#   visible_bias_(t)  =     visible_bias_(t) +  sum( input - activated_visivle_at_k_step_sample )  +  visible_bias_(t-1)\n",
    "#   hidden_bias_(t)   =     hidden_bias_(t)  +  sum( activated_initial_hidden - activated_hidden_at_k_step_sample ) + hidden_bias_(t-1)\n",
    "    def update_parameters(self, g_0, g_k, v, p_v_k, p_h_0, p_h_k):\n",
    "        self.w_moment *= self.momentum\n",
    "        del_w          = (g_0 - g_k) + self.w_moment\n",
    "\n",
    "        self.a_moment *= self.momentum\n",
    "        del_a          = torch.sum(v - p_v_k, dim=0) + self.a_moment\n",
    "\n",
    "        self.b_moment *= self.momentum\n",
    "        del_b          = torch.sum(p_h_0 - p_h_k, dim=0) + self.b_moment\n",
    "\n",
    "        batch_size = v.size(0)\n",
    "\n",
    "        self.w += del_w * self.alpha / batch_size\n",
    "        self.a += del_a * self.alpha / batch_size\n",
    "        self.b += del_b * self.alpha / batch_size\n",
    "\n",
    "        self.w -= (self.w * self.weight_decay)\n",
    "        \n",
    "        self.w_moment = del_w\n",
    "        self.a_moment = del_a\n",
    "        self.b_moment = del_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"/home/pushpull/mount/intHdd/dataset/mnist/mnist_train.csv\", header=None)\n",
    "data    = torch.tensor(np.array(dataset)[:, 1:], device=\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = RBM()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist.data = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist.number_features = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Number  :  0        Error  :  527743.1875\n",
      "Epoch Number  :  1        Error  :  283213.125\n",
      "Epoch Number  :  2        Error  :  261696.796875\n",
      "Epoch Number  :  3        Error  :  258503.1875\n",
      "Epoch Number  :  4        Error  :  257862.4375\n",
      "Epoch Number  :  5        Error  :  258103.671875\n",
      "Epoch Number  :  6        Error  :  258327.65625\n",
      "Epoch Number  :  7        Error  :  258373.75\n",
      "Epoch Number  :  8        Error  :  258460.90625\n",
      "Epoch Number  :  9        Error  :  258357.0625\n",
      "Epoch Number  :  10        Error  :  258420.140625\n",
      "Epoch Number  :  11        Error  :  258400.65625\n",
      "Epoch Number  :  12        Error  :  258748.265625\n",
      "Epoch Number  :  13        Error  :  258971.53125\n",
      "Epoch Number  :  14        Error  :  258795.984375\n"
     ]
    }
   ],
   "source": [
    "error = mnist.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}