NormalNaiveBayes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from math import pi\n",
    "np.set_printoptions(precision=3, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class GaussianBinaryNaiveBayes:\n",
    "    means, variances = None, None\n",
    "    log_prior = None\n",
    "    num_feat = None\n",
    "    labels = None\n",
    "    dropped_columns = None\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def fit(self, X, y, standardize=False):\n",
    "        self.num_feat = X.shape[1]\n",
    "        self.labels = np.unique(y)\n",
    "        self.dropped_columns = np.where(X.sum(axis=0) == 0)\n",
    "        X_ = np.delete(X, self.dropped_columns, axis=1)  # drop columns where features are all 0\n",
    "\n",
    "        if standardize:  # Mean center and normalize data\n",
    "            X_ = (X_ - X_.mean(axis=0))/X_.std(axis=0)  \n",
    "\n",
    "        self.means, self.variances = np.empty(shape=(2, self.labels.size, X_.shape[1]))\n",
    "#         variances = X_.std(axis=0)**2\n",
    "        for index, label in enumerate(self.labels):\n",
    "            self.means[index] = X_[y==self.labels[index]].mean(axis=0)\n",
    "            self.variances[index] = (X_[y==self.labels[index]].std(axis=0))**2\n",
    "#         self.wki = (means[0] - means[1])/variances\n",
    "#         print(self.wki.shape)\n",
    "        self.log_prior = np.log(np.bincount(y)[self.labels]/y.shape[0])\n",
    "#         self.wk0 = np.log(prior[0]/prior[1]) + (np.sum((means[0]**2 - means[1]**2)/(2*variances)))\n",
    "    \n",
    "    def predict(self, X):\n",
    "        X_ = np.delete(X, self.dropped_columns, axis=1)\n",
    "#         num = (self.wk0 + np.sum(self.wki.reshape(1,-1) * X_, axis=1)) \n",
    "#         num = np.argmax(num, axis=0)\n",
    "#         print(num)\n",
    "        log_class_cond_prob = np.zeros(shape=(self.labels.size, X.shape[0]))\n",
    "        for index in range(self.labels.size):\n",
    "            log_likelihood = - 0.5 * np.sum(np.log(2 * np.pi * self.variances[index, :]))\n",
    "            value = ((X_ - self.means[index, :]) ** 2)/(self.variances[index, :])\n",
    "            value = value[:,~np.isnan(value).any(axis=0)]\n",
    "            log_likelihood -= 0.5 * np.sum(value, axis=1)\n",
    "            log_class_cond_prob[index] = self.log_prior[index] + log_likelihood\n",
    "        log_class_cond_prob = log_class_cond_prob.T\n",
    "        return self.labels[np.argmax(log_class_cond_prob, axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load MNIST train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_data = np.genfromtxt('../Data/MNIST_HW1/train.csv', delimiter=',', dtype=np.float)\n",
    "mnist_data = mnist_data.T\n",
    "X = mnist_data[:, :-1]\n",
    "y = mnist_data[:, -1].astype(int)\n",
    "# Select data corresponding to 0 and 1 digits\n",
    "rows_0_1 = ((y == 0) + (y == 1))\n",
    "# Select data corresponding to 3 and 5 digits\n",
    "rows_3_5 = ((y == 3) + (y == 5))\n",
    "X_0_1 = X[rows_0_1]\n",
    "X_3_5 = X[rows_3_5]\n",
    "y_0_1 = y[rows_0_1]\n",
    "y_3_5 = y[rows_3_5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit to model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nbc_0_1 = GaussianBinaryNaiveBayes()\n",
    "nbc_0_1.fit(X_0_1, y_0_1)\n",
    "nbc_3_5 = GaussianBinaryNaiveBayes()\n",
    "nbc_3_5.fit(X_3_5, y_3_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load MNIST test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mnist_test_data = np.genfromtxt('../Data/MNIST_HW1/test.csv', delimiter=',', dtype=np.float)\n",
    "mnist_test_data = mnist_test_data.T\n",
    "X_test = mnist_test_data[:, :-1]\n",
    "y_test = mnist_test_data[:, -1].astype(int)\n",
    "# Select data corresponding to 0 and 1 digits\n",
    "rows_0_1 = ((y_test == 0) + (y_test == 1))\n",
    "# Select data corresponding to 3 and 5 digits\n",
    "rows_3_5 = ((y_test == 3) + (y_test == 5))\n",
    "X_test_0_1 = X_test[rows_0_1]\n",
    "X_test_3_5 = X_test[rows_3_5]\n",
    "y_test_0_1 = y_test[rows_0_1]\n",
    "y_test_3_5 = y_test[rows_3_5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/ipykernel/__main__.py:36: RuntimeWarning: divide by zero encountered in log\n",
      "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/ipykernel/__main__.py:37: RuntimeWarning: divide by zero encountered in true_divide\n",
      "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/ipykernel/__main__.py:37: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    }
   ],
   "source": [
    "y_pred_0_1 = nbc_0_1.predict(X_test_0_1)\n",
    "y_pred_3_5 = nbc_3_5.predict(X_test_3_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Wine Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following function just samples the requested number of samples from a given bool_array. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample_mask_from(bool_array, sample_size):\n",
    "    sample_rows_mask = bool_array.copy()\n",
    "    temp = bool_array[bool_array == True]\n",
    "    temp[sample_size:] = False\n",
    "    np.random.shuffle(temp)\n",
    "    sample_rows_mask[sample_rows_mask == True] = temp\n",
    "    return sample_rows_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load wine data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wine_data = np.genfromtxt('../Data/wine.data', delimiter=',')\n",
    "rows_1 = (wine_data[:,0] == 1)\n",
    "rows_2 = (wine_data[:,0] == 2)\n",
    "wine_data = wine_data[rows_1 + rows_2]\n",
    "rows_1 = (wine_data[:,0] == 1)\n",
    "rows_2 = (wine_data[:,0] == 2)\n",
    "five_sample_rows_mask = sample_mask_from(rows_1, 5) + sample_mask_from(rows_2, 5)\n",
    "train_set_1 = wine_data[five_sample_rows_mask]\n",
    "test_set_1 = wine_data[~five_sample_rows_mask]\n",
    "\n",
    "fifty_sample_rows_mask = sample_mask_from(rows_1, 50) + sample_mask_from(rows_2, 50)\n",
    "train_set_2 = wine_data[fifty_sample_rows_mask]\n",
    "test_set_2 = wine_data[~fifty_sample_rows_mask]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nbc_w_1 = GaussianBinaryNaiveBayes()\n",
    "nbc_w_1.fit(train_set_1[:, 1:], train_set_1[:, 0].astype(int))\n",
    "nbc_w_2 = GaussianBinaryNaiveBayes()\n",
    "nbc_w_2.fit(train_set_2[:, 1:], train_set_2[:, 0].astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w_pred_lab_1, w_pred_lab_2 = nbc_w_1.predict(test_set_1[:, 1:]), nbc_w_2.predict(test_set_2[:, 1:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate model performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/sklearn/utils/fixes.py:64: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  if 'order' in inspect.getargspec(np.copy)[0]:\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score, classification_report\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wine Data:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    For five samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class 1</th>\n",
       "      <th>Class 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class 1</th>\n",
       "      <td>47</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class 2</th>\n",
       "      <td>9</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Class 1  Class 2\n",
       "Class 1       47        7\n",
       "Class 2        9       57"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(test_set_1[:, 0], w_pred_lab_1), columns=['Class 1', 'Class 2'], index = ['Class 1', 'Class 2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        Accuracy Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 86.7%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.3}%\".format(accuracy_score(test_set_1[:, 0], w_pred_lab_1, normalize=True) * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        Other metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "    Class 1       0.84      0.87      0.85        54\n",
      "    Class 2       0.89      0.86      0.88        66\n",
      "\n",
      "avg / total       0.87      0.87      0.87       120\n",
      "\n"
     ]
    }
   ],
   "source": [
    "target_names = ['Class 1', 'Class 2']\n",
    "print(classification_report(test_set_1[:, 0], w_pred_lab_1, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    For fifty samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class 1</th>\n",
       "      <th>Class 2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Class 1</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Class 2</th>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Class 1  Class 2\n",
       "Class 1        9        0\n",
       "Class 2        0       21"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(confusion_matrix(test_set_2[:, 0], w_pred_lab_2), columns=['Class 1', 'Class 2'], index = ['Class 1', 'Class 2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        Accuracy Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 1e+02%\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.3}%\".format(accuracy_score(test_set_2[:, 0], w_pred_lab_2, normalize=True) * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "        Other Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "    Class 1       1.00      1.00      1.00         9\n",
      "    Class 2       1.00      1.00      1.00        21\n",
      "\n",
      "avg / total       1.00      1.00      1.00        30\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(test_set_2[:, 0], w_pred_lab_2, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "K fold cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_folds = 10\n",
    "skf = StratifiedKFold(wine_data[:, 0], n_folds=num_folds, shuffle=True)\n",
    "X = wine_data[:, 1:]\n",
    "y = wine_data[:, 0].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "accuracy_scores = []\n",
    "for train_index, test_index in skf:\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    clf = GaussianBinaryNaiveBayes()\n",
    "    clf.fit(X_train, y_train.astype(int))\n",
    "    y_pred = clf.predict(X_test)\n",
    "    accuracy_scores.append(accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Average accuracy from cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is: 98.46 %\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy is: {:.4} %\".format(np.asarray(accuracy_scores).mean()*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}