Q3- Ridge Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def findCommunities(filename):\n",
    "    communities = []\n",
    "    with open(filename, 'r') as csvfile:\n",
    "        filereader = csv.reader(csvfile)\n",
    "        j = 0\n",
    "        for i in filereader:\n",
    "            if j > 0:\n",
    "                if i[1] not in communities:\n",
    "                    communities.append(i[1])\n",
    "            j+=1\n",
    "    return communities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extractData(filename,communities,addstate = False,addcommunity = False):\n",
    "    features = []\n",
    "    x=[]\n",
    "    y=[]\n",
    "    with open(filename, 'r') as csvfile:\n",
    "        filereader = csv.reader(csvfile)\n",
    "        j = 0\n",
    "        for i in filereader:\n",
    "            if j > 0:\n",
    "                data = i[2:len(i)-1]\n",
    "                if addstate:\n",
    "                    state = [0.0]*56\n",
    "                    state[int(i[0])-1] = 1.0\n",
    "                    data = data + state\n",
    "                    features = features + [\"state\"+str(i) for i in range(56)]\n",
    "                if addcommunity:\n",
    "                    tmp = [0.0]*len(communities)\n",
    "                    tmp[communities.index(i[1])] = 1.0\n",
    "                    data = data + tmp\n",
    "                    features = features + communities\n",
    "                vect = np.array(data)\n",
    "                #vect = np.array(i[3:len(i)-1])\n",
    "                x.append(vect.astype(np.float))\n",
    "                y.append(float(i[-1]))\n",
    "            else:\n",
    "                features = [i[0]]+i[2:len(i)-1]\n",
    "            j+=1\n",
    "        y = np.array(y)\n",
    "        x = np.array(x)\n",
    "        \n",
    "        return x,y,features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def meanSquaredError(true,predict):\n",
    "    print(\"Mean squared error: %.4f\"\n",
    "      % np.mean((predict-true) ** 2))\n",
    "\n",
    "def ridgeRegression(alpha = 1.0):\n",
    "    return Ridge(alpha=alpha, fit_intercept=True, normalize=False, copy_X=True,\n",
    "                 max_iter=None, tol=0.001, solver='auto', random_state=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def do_cross_validation(X, y,clf, n_folds=5):\n",
    "    cv = KFold(len(y), n_folds)\n",
    "    accuracies = []\n",
    "    for train_ind, test_ind in cv: \n",
    "        clf.fit(X[train_ind], y[train_ind])\n",
    "        predictions = clf.predict(X[test_ind])\n",
    "        meanSquaredError(y[test_ind],predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testAlpha(alpha_list):\n",
    "    for i in alpha_list:\n",
    "        ridgeReg= ridgeRegression(alpha = i)\n",
    "        ridgeReg.fit(X[:1000],Y[:1000])\n",
    "        predictedY=ridgeReg.predict(X[1000:])\n",
    "        meanSquaredError(predictedY,Y[1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "filename = 'C:\\\\Users\\\\ar1\\\\Documents\\ML\\\\Project\\\\Crime Prediction Data(1)\\\\Crime Prediction Data\\\\communities-crime-clean.csv'\n",
    "distintCommunities = findCommunities(filename)\n",
    "X,Y,Features = extractData(filename,distintCommunities,addstate=True,addcommunity = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ridgeReg= ridgeRegression()\n",
    "ridgeReg.fit(X[:1000],Y[:1000])\n",
    "predictedY=ridgeReg.predict(X[1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error: 0.0172\n",
      "Mean squared error: 0.0386\n",
      "Mean squared error: 0.0244\n",
      "Mean squared error: 0.0170\n",
      "Mean squared error: 0.0191\n",
      "Mean squared error: 0.0132\n",
      "Mean squared error: 0.0312\n",
      "Mean squared error: 0.0266\n",
      "Mean squared error: 0.0259\n",
      "Mean squared error: 0.0274\n",
      "Mean squared error: 0.0122\n"
     ]
    }
   ],
   "source": [
    "meanSquaredError(predictedY,Y[1000:])\n",
    "do_cross_validation(X[:1000],Y[:1000],ridgeReg,10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explaination : MSE under 10 fold CV\n",
    "3-c-i) The MSE under 10 fold CV can be found above. The MSE under 10 fold CV, when plotted on a graph, turns out to have its highs and lows. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error: 0.0166\n",
      "Mean squared error: 0.0172\n",
      "Mean squared error: 0.0194\n",
      "Mean squared error: 0.0251\n",
      "Mean squared error: 0.0291\n"
     ]
    }
   ],
   "source": [
    "alphaList = [10,1,0.1,.01,.001]\n",
    "testAlpha(alphaList)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explaination : Best Alpha\n",
    "2-b-iii) The best alpha is 10 with a score of 0.0166. This is the best score compared with the other alpha scores."
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Extra: \n",
    "Note: The MSE below was tried out with some additional features like fold. I just wanted to see if the MSE would turn out to better with adding/removing features while prediciting.(PLease consider the above MSE and not the below results as I was able to find the best alpha,10, with a score of 0.166 )\n",
    "\n",
    "Mean squared error: 0.0177\n",
    "Mean squared error: 0.0176\n",
    "Mean squared error: 0.0183\n",
    "Mean squared error: 0.0187\n",
    "Mean squared error: 0.0188\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error: 0.0047\n"
     ]
    }
   ],
   "source": [
    "ridgeReg= ridgeRegression()\n",
    "ridgeReg.fit(X,Y)\n",
    "predictedY=ridgeReg.predict(X)\n",
    "meanSquaredError(predictedY,Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Explaination : MSE on the training set\n",
    "3-c-ii) The MSE on the training set is 0.0047. The MSE here is really good and it means that I was able to find out a very good fit ( because, lower the MSE, the better the fit is ). The MSE on the training set for ridgeRegression is better than the MSE on the training set for Linear Regression. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Irvinecity 0.366471328241\n",
      "Greenvillecity 0.304151822622\n",
      "WestCovinacity 0.272955031258\n",
      "Northportcity 0.249991072666\n",
      "Jonesborocity 0.241202621618\n",
      "Plantationcity 0.231473526834\n",
      "Coltoncity 0.222102608709\n",
      "Prichardcity 0.220073174043\n",
      "Atholtown 0.212138537445\n",
      "Bentonvillecity 0.206723825867\n",
      "\n",
      "\n",
      "---------------------\n",
      "\n",
      "\n",
      "Sylacaugacity -0.206712082519\n",
      "Eustiscity -0.193520397373\n",
      "Selmacity -0.183195306494\n",
      "Alpharettacity -0.173953640364\n",
      "state26 -0.171897677954\n",
      "Natchitochescity -0.157725946388\n",
      "Lawrencevillecity -0.152652367071\n",
      "Conwaycity -0.148420041155\n",
      "CollegeParkcity -0.148220007261\n",
      "Atlantacity -0.140725838344\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print (\"Important  \\n\\n\")\n",
    "\n",
    "feature_indx = np.argsort(ridgeReg.coef_)[::-1]\n",
    "for i in feature_indx[:10]:\n",
    "    print (Features[i], ridgeReg.coef_[i])\n",
    "\n",
    "\n",
    "print (\"\\n\\n---------------------\\n\\n\")\n",
    "\n",
    "feature_indx = np.argsort(ridgeReg.coef_)\n",
    "for i in feature_indx[:10]:\n",
    "    print (Features[i], ridgeReg.coef_[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}