Machine Learning with Scikit-Learn - 44 - Automatic Feature Selection - 3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# <center> Automatic Feature Selection </center>\n",
    "\n",
    "- <b><span style='color:green'>to reduce dimensionality<span></b>\n",
    "- common methods: univariate statistics, model-based selection, iterative selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 1. Univariate Statistics\n",
    "\n",
    "- determines the relationship between each feature and output (target)\n",
    "- only the features with highest confidence are selected\n",
    "- <b><span style='color:blue'>SelectKBest</span></b> - selecting K number of features\n",
    "- <b><span style='color:blue'>SelectPercentile</span></b> - selection is made based on a percentage of the original features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape is: (284, 80)\n",
      "X_train_selected.shape is: (284, 40)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectPercentile\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "%matplotlib inline\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "\n",
    "rng = np.random.RandomState(42)\n",
    "noise = rng.normal(size=(len(cancer.data), 50))\n",
    "\n",
    "X_w_noise = np.hstack([cancer.data, noise])\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_w_noise, cancer.target, random_state=0, test_size=.5)\n",
    "\n",
    "select = SelectPercentile(percentile=50)\n",
    "select.fit(X_train, y_train)\n",
    "X_train_selected = select.transform(X_train)\n",
    "\n",
    "print('X_train.shape is: {}'.format(X_train.shape))\n",
    "print('X_train_selected.shape is: {}'.format(X_train_selected.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ True  True  True  True  True  True  True  True  True False  True False\n",
      "  True  True  True  True  True  True False False  True  True  True  True\n",
      "  True  True  True  True  True  True False False False  True False  True\n",
      " False False  True False False False False  True False False  True False\n",
      " False  True False  True False False False False False False  True False\n",
      "  True False False False False  True False  True False False False False\n",
      "  True  True False  True False False False False]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1b4db64d208>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA60AAAAvCAYAAAAINSyRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACV9JREFUeJzt3WmMJGUdx/Hvz103i3hwrUhAQCKRbKICi0QUDYIHogES\nDREhIUbDGw+IGkVNvEkwxjOiBkFFxQNRlBCjIIIYX6A7HhFEzgBCgF3RFQ8Cgn9fVG12Mu7szGxv\nb9fT+/0kk+mqrql+pn/Vz/S/63lqUlVIkiRJkjREj5t0AyRJkiRJmo9FqyRJkiRpsCxaJUmSJEmD\nZdEqSZIkSRosi1ZJkiRJ0mBZtEqSJEmSBqupojXJsUluSnJrkrMm3R7NL8mXk6xLcv2sdbsluTLJ\nLf33XSfZRs0vydOTXJ3kj0luSHJGv94MBy7JyiS/SvL7PrsP9eufkeS6vv/8TpIVk26r5pdkWZLf\nJrm8Xza/RiS5I8kfkvwuydp+nX1nA5LskuSSJH9KcmOSI8yuDUme1b/mNn49mORM85sezRStSZYB\n5wKvBFYDJydZPdlWaQu+Chw7Z91ZwFVVdSBwVb+sYXoUeEdVrQaeD7y5f72Z4fA9DBxdVc8FDgaO\nTfJ84GPAp6rqmcDfgDdOsI1a2BnAjbOWza8tL6mqg6vqsH7ZvrMNnwF+XFUHAc+lew2aXQOq6qb+\nNXcwsAb4N3Ap5jc1milagcOBW6vq9qp6BPg2cMKE26R5VNW1wF/nrD4BuLC/fSFw4nZtlBatqu6t\nqt/0t/9B94d7b8xw8Krzz37x8f1XAUcDl/TrzW7AkuwDvAo4v18O5tc6+86BS/IU4MXABQBV9UhV\nbcDsWnQMcFtV3Yn5TY2Wita9gT/PWr67X6d27FlV9/a37wP2nGRjtDhJ9gcOAa7DDJvQDy39HbAO\nuBK4DdhQVY/2m9h/DtungXcB/+2Xd8f8WlLAFUlmkpzer7PvHL5nAOuBr/RD889PsjNm16LXAd/q\nb5vflGipaNUUqaqi+8OuAUvyROB7wJlV9eDs+8xwuKrqsX6I1D50o1QOmnCTtEhJXg2sq6qZSbdF\nW+3IqjqUbjrTm5O8ePad9p2DtRw4FPhCVR0C/Is5Q0nNbvj6+f7HA9+de5/5ta2lovUe4Omzlvfp\n16kd9yfZC6D/vm7C7dEWJHk8XcF6UVV9v19thg3ph7ZdDRwB7JJkeX+X/edwvRA4PskddNNgjqab\nZ2d+jaiqe/rv6+jm1B2OfWcL7gburqrr+uVL6IpYs2vLK4HfVNX9/bL5TYmWitZfAwf2V1BcQXfq\n/7IJt0lLcxlwWn/7NOCHE2yLtqCfQ3cBcGNVfXLWXWY4cElWJdmlv70T8DK6OclXA6/tNzO7gaqq\n91TVPlW1P93fuZ9V1SmYXxOS7JzkSRtvAy8Hrse+c/Cq6j7gz0me1a86BvgjZteak9k0NBjMb2qk\nO1PehiTH0c31WQZ8uarOnnCTNI8k3wKOAvYA7gc+APwAuBjYF7gTOKmq5l6sSQOQ5EjgF8Af2DSv\n7r1081rNcMCSPIfuYhPL6D6YvLiqPpzkALozd7sBvwVOraqHJ9dSLSTJUcA7q+rV5teGPqdL+8Xl\nwDer6uwku2PfOXhJDqa7ANoK4HbgDfT9KGY3eP0HRXcBB1TV3/t1vvamRFNFqyRJkiRpx9LS8GBJ\nkiRJ0g7GolWSJEmSNFgWrZIkSZKkwbJolSRJkiQNlkWrJEmSJGmwRipak+yW5Mokt/Tfd93Ctk9O\ncneSz434mKeP8vOaLPNrl9m1zfzaZXZtM7+2mV+7zG66jHqm9Szgqqo6ELiqX57PR4BrR3w8AA/A\ntplfu8yubebXLrNrm/m1zfzaZXZTZNSi9QS6f2JP//3EzW2UZA2wJ3DFiI8nSZIkSdqBpKq2/oeT\nvwO/AvYH7gAOr6qnzNnmEOAa4D7gCcANVXXsIva99Q3TNrdmzZolbT8zMzOWfS9lv9NuqZksls9x\n2+Y7LtavX8+qVau2SxuWegyNqw8YwmtkW7Rhe2YH7T3HQ2nHUl57Q3iOWzTtfcs4DeF92fZ+3rZF\n3zmu90TT8hxvCzMzM3+pqgWDWrBoTfJT4Gmbuet9wHeAD1bVOUnO6m+vnPPzHwJWVtW7k5wJnA3s\nXVUbNvNYp7PpVH57z/oUW+qHG0nGsu+l7HfajfKB05b4HLdtXMfFUiz1GBpXHzCE18gQ8liqIfx+\n4zyGxtWO1o7jFk173zJOHstbZ1zviXyON0kyU1WHLbjdiGdaHwGeB+wFfBHYD3hPVZ0za5uLgBcB\nuwMr6YYkf7OqTllg3+0961PMonV4hvAmTcMzhD9Y0/7GcghtGKch/H4WrVu332k37X3LOHksbx2L\n1vFbbNE66pzW/wLHAecCFwMPAycnWb1xg744PQf4OvBR4F/A8nkafXqStUnWjtguSZIkSdIUGHV4\n8NeAm4FnA4/RnUm9EriVbkjwm/p9/IluPutewN/oitbdawsP7pnWYfFM6/AM4cyChmcIn7JO+9mQ\nIbRhnIbw+3mmdev2O+2mvW8ZJ4/lreOZ1vFb7JnWzZ7xnK2qXrqFB7kPOA/4BPAyuiJ2NfDAnOG/\nK4EHgbcDq4CP0w0X/suc/c2e0ypJkiRJ2sGNOjz4MuA1dMOCLweeCmwADti4QZIVwB7Ad4ECPs88\nxXJVnVdVhy2m2pYkSZIkTb9Ri9ZzgGfSFaU3AwcB+wK7JTm/3+YkuqHBp9L9L9dH+/UPjPjYkiRJ\nkqQpN1LRWlUPAF+hm896Kt18VYCHNs5nrapvAG8FHqK7wvB/gF9ubj6rF2KSJEmSJM026plW6Ib8\nbgB+AtwI3AU8lOTDSY7vt5mhuwjT2+gK3M9udkcOD5YkSZIkzbLghZgW4VZgBfAK4J5++fqqej9A\nkscBnwP+DbwA+FK/3f/xQkySJEmSpNkW/Jc3C+4gOYKuKH0SsIyuaL0G2AlYC/wcWN/f9xhdofwI\ncGRVzTsMOMl64M7N3LUHc646rKaYX7vMrm3m1y6za5v5tc382mV2bdivqlYttNG2KFqX012E6Ri6\nM6i/Bl5fVTfMs/01wDu3VLAu8HhrHT7cLvNrl9m1zfzaZXZtM7+2mV+7zG66jDyntaoeBd7Cpjmt\nF1fVDXPmtEqSJEmStGTbYk4rVfUj4Edz1r1/nm2P2haPKUmSJEmaftvi6sHb23mTboBGYn7tMru2\nmV+7zK5t5tc282uX2U2Rkee0SpIkSZI0Li2eaZUkSZIk7SAsWiVJkiRJg2XRKkmSJEkaLItWSZIk\nSdJgWbRKkiRJkgbLolWSJEmSNFj/A5QS5b4SZ+dnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b4db5f8780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mask = select.get_support()\n",
    "print(mask)\n",
    "plt.matshow(mask.reshape(1,-1), cmap='gray_r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The score of Logistic Regression on all features: 0.930\n",
      "The score of Logistic Regression on the selected features: 0.940\n"
     ]
    }
   ],
   "source": [
    "X_test_selected = select.transform(X_test)\n",
    "logreg = LogisticRegression()\n",
    "logreg.fit(X_train, y_train)\n",
    "print('The score of Logistic Regression on all features: {:.3f}'.format(logreg.score(X_test, y_test)))\n",
    "\n",
    "logreg.fit(X_train_selected, y_train)\n",
    "print('The score of Logistic Regression on the selected features: {:.3f}'.format(logreg.score(X_test_selected, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "### 2. Model-Based Feature Selection\n",
    "\n",
    "- uses a supervised model to determine the importance of each feature\n",
    "- keeps the most important features\n",
    "- needs a measure for the importance of features (DT and RF have the 'feature_importances' attribute)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "select = SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The shape of X_train is:  (284, 80)\n",
      "The shape of X_train_s is  (284, 40)\n"
     ]
    }
   ],
   "source": [
    "select.fit(X_train, y_train)\n",
    "X_train_s = select.transform(X_train)\n",
    "print('The shape of X_train is: ', X_train.shape)\n",
    "print('The shape of X_train_s is ', X_train_s.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x1b4db67a240>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA60AAAA4CAYAAAAW5h6yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADN9JREFUeJzt3X+QXWV9x/H3h/yYFUJBQvghCBGIMimUhGDGIDABKkZK\nA62MbQoz0NKm06FWaKHFMrWAMIZRS7ViK8UALT8EorYUHUmKoFQ7CVkJkJjwyyYaCiSiEQM0afDT\nP86Tye26m/1xd91z7n5eMzv3nOc89zzP3u+55+73nuc5K9tERERERERE1NEeo92BiIiIiIiIiL4k\naY2IiIiIiIjaStIaERERERERtZWkNSIiIiIiImorSWtERERERETUVpLWiIiIiIiIqK1GJa2S5kl6\nStKzkq4Y7f5E3yQtlrRJ0uqWsv0kLZP0THl882j2Mfom6a2SHpL0XUlrJH2olCeGNSepS9IKSY+X\n2F1dyt8maXk5f94taeJo9zX6JmmcpMck3V/WE7+GkLRe0pOSVklaWcpy7mwASftKWiJpnaS1kuYk\nds0g6R3lPbfz5xVJlyR+naMxSaukccCNwPuA6cACSdNHt1exG7cC83qUXQE8aHsa8GBZj3raAfyZ\n7enAu4CLy/stMay/bcBpto8DZgDzJL0LuB64wfZRwI+Bi0axj9G/DwFrW9YTv2Y51fYM2yeU9Zw7\nm+FTwNdsHw0cR/UeTOwawPZT5T03A5gFvAZ8mcSvYzQmaQVmA8/a/p7t7cAXgLNHuU/RB9vfBH7U\no/hs4LayfBtwzi+0UzFgtl+w/Z2y/FOqD+5DSAxrz5WtZXVC+TFwGrCklCd2NSbpUODXgJvLukj8\nmi7nzpqTtA9wCvB5ANvbbW8hsWui04HnbG8g8esYTUpaDwF+0LK+sZRFcxxo+4Wy/CJw4Gh2JgZG\n0lRgJrCcxLARytDSVcAmYBnwHLDF9o5SJefPevtb4M+Bn5X1ySR+TWJgqaRuSQtLWc6d9fc2YDNw\nSxmaf7OkvUjsmui3gbvKcuLXIZqUtEYHsW2qD/aoMUmTgC8Cl9h+pXVbYlhftt8oQ6QOpRqlcvQo\ndykGSNJZwCbb3aPdlxiyk2wfTzWd6WJJp7RuzLmztsYDxwN/b3sm8Co9hpImdvVX5vvPB+7tuS3x\na7YmJa3PA29tWT+0lEVzvCTpYIDyuGmU+xO7IWkCVcJ6h+0vleLEsEHK0LaHgDnAvpLGl005f9bX\nu4H5ktZTTYM5jWqeXeLXELafL4+bqObUzSbnzibYCGy0vbysL6FKYhO7Znkf8B3bL5X1xK9DNClp\nfRSYVu6gOJHq0v99o9ynGJz7gAvK8gXAv45iX2I3yhy6zwNrbf9Ny6bEsOYkTZG0b1l+E/AeqjnJ\nDwHnlmqJXU3Z/rDtQ21Ppfqc+7rt80j8GkHSXpL23rkMnAGsJufO2rP9IvADSe8oRacD3yWxa5oF\n7BoaDIlfx1B1pbwZJJ1JNddnHLDY9nWj3KXog6S7gLnA/sBLwF8D/wLcAxwGbAA+YLvnzZqiBiSd\nBDwCPMmueXV/STWvNTGsMUm/QnWziXFUX0zeY/saSUdQXbnbD3gMON/2ttHrafRH0lzgMttnJX7N\nUOL05bI6HrjT9nWSJpNzZ+1JmkF1A7SJwPeA36WcR0nsaq98UfR94AjbPyllee91iEYlrRERERER\nETG2NGl4cERERERERIwxSVojIiIiIiKitpK0RkRERERERG0laY2IiIiIiIjaStIaERERERERtdVW\n0ippP0nLJD1THt+8m7q/JGmjpM+02ebCdp4foyvxa67ErtkSv+ZK7Jot8Wu2xK+5ErvO0u6V1iuA\nB21PAx4s6335KPDNNtsDyAHYbIlfcyV2zZb4NVdi12yJX7Mlfs2V2HWQdpPWs6n+iT3l8ZzeKkma\nBRwILG2zvYiIiIiIiBhDZHvoT5Z+AqwApgLrgdm29+lRZybwMPAisCewxva8Aex76B2LRpk1a9aA\n63Z3d3dsHyKGQ1/H8ubNm5kyZcr/KxvMsTyY98hg1aUfo22w55aRei1GKh4jee6sw3FRl+O4Dp+T\nnaTO585ONhyvcW+xG0k5Loamu7v7h7b7DVS/SaukfwcO6mXTlcDdwFW2F0m6oix39Xj+1UCX7b+Q\ndAlwHXCI7S29tLWQXZfyE80xYjBfnEjq2D5EDIeROpbb+YKzKf0YbYM9t4zUazFS8RjJc2cdjou6\nHMd1+JzsdHWJdSdr4mvcxD7XgaRu2yf0W6/NK63bgXcCBwP/ABwOfNj2opY6dwAnA5OBLqohyXfa\nPq+ffSeaY0Qd/uipQx8ihkOS1uZK0jp0dTgu6nIc1+FzstPVJdadrImvcRP7XAcDTVrbndP6M+BM\n4EbgHmAbsEDS9J0VSnK6CPhn4FrgVWB8H51eKGmlpJVt9isiIiIiIiI6QLvDg/8JeBo4FniD6krq\nMuBZqiHBv1/2sY5qPuvBwI+pktbJ3k3judI6dtThm/o69CFiOORKa3PlSuvQ1eG4qMtxXIfPyU5X\nl1h3sia+xk3scx0M9Eprr1c8W9n+1d008iJwE/BJ4D1USex04OUew3+7gFeAPwWmAB+nGi78wx77\na53TGhEREREREWNcu8OD7wPeTzUs+H7gAGALcMTOCpImAvsD9wIGPksfybLtm2yfMJBsOyIiIiIi\nIjpfu0nrIuAoqqT0aeBo4DBgP0k3lzofoBoafD7V/3LdUcpf7rmzzGmNiIiIiIiIVm0lrbZfBm6h\nms96PtV8VYDXd85ntX078EHgdao7DP8v8K3e5rPmSmtERERERES0avdKK1RDfrcADwBrge8Dr0u6\nRtL8Uqeb6iZMf0KV4H56GNqNiIiIiIiIDtfvjZgG4FlgIvBe4Pmyvtr2RwAk7QF8BngNOBH4x1Lv\n5+RGTBEREREREdGq33950+8OpDlUSenewDiqpPVh4E3ASuAbwOay7Q2qRHk7cJLtPueuStoMbOhl\n0/70uOtwNEri11yJXbMlfs2V2DVb4tdsiV9zJXbNcLjtKf1VGo6kdTzVTZhOp7qC+ijwO7bX9FH/\nYeCy3SWs/bS3MnNemyvxa67ErtkSv+ZK7Jot8Wu2xK+5ErvO0vacVts7gD9m15zWe2yv6TGnNSIi\nIiIiImLQhmNOK7a/Cny1R9lH+qg7dzjajIiIiIiIiM43HHcP/kW7abQ7EG1J/JorsWu2MRs/SVsH\nWX+upPtHqj+ljbskPSHp0h7lV0l6XtKq8rOIIcRO0jmSpg9bh6MdY/a91yESv+ZK7DpI23NaIyIi\n6kzSVtuTBlF/LtW9F84aof4cBCy3fXgv264Cttr+RJtt3Arcb3vJIJ4zvkz5iYiIqJUmXmmNiIgY\ntHIF9WFJSyStk3SHJJVt80rZfwC/2fKcvSQtlrRC0mOSzi7ll0paXJaPlbRa0p492uuSdIukJ8tz\nTy2blgIHlCupJw+w77MkfUNSt6QHJB1cyv9A0qOSHpf0RUl7SjoRmA98vLRxZPm9TyjP2V/S+rJ8\noaR7Jf1b6ReSLi/7fELS1S2vw1dKO6sl/daQghARETEESVojImIsmQlcAkwHjgDeLamL6n+I/zpw\nMnBQS/0rga/bng2cSpUI7gV8CjhK0m8AtwB/aPu1Hm1dDNj2scAC4LbS1nzgOdszbD/SSx8vbRke\n/F5JE4C/A861PQtYDFxX6n7J9jttH0d1M8SLbH8buA+4vLTxXD+vyRzgAtunSToDmAbMBmYAsySd\nAswD/tv2cbaPAb7Wzz4jIiKGzbDciCkiIqIhVtjeCCBpFTAV2Ar8l+1nSvntwMJS/wxgvqTLynoX\ncJjttZIuBJ4APmf7W720dRJVsontdZI2AG8HXumnjze0Dg+WdAxwDLCsXBgeB7xQNh8j6VpgX2AS\n1Z38B2uZ7R+V5TPKz2NlfRJVEvsI8ElJ11MNO+4t2Y6IiBgRSVojImIs2day/Ab9fw4KeL/tp3rZ\nNo0q4X3LMPVtd31YY3tOL9tuBc6x/XhJouf2sY8d7Bpd1dVj26s92vqY7c/9XCek44EzgY9JWmr7\nmgH/BhEREW3I8OCIiBjr1gFTJR1Z1he0bHsA+GDL3NeZ5XEf4NPAKcBkSef2st9HgPNK/bcDhwG9\nJb/9eQqYImlO2dcESb9ctu0NvFCGEJ/X8pyflm07rQdmleXe+rrTA8DvSZpU2jpE0gGS3gK8Zvt2\n4BPA8UP4PSIiIoYkSWtERIxptv+HajjwV8qNmDa0bP4oMAF4QtKasg5wA3Cj7aeBi4BFkg7osevP\nAntIehK4G7jQ9jYGyfZ2qkTzekmPA6uAE8vmvwKWA8uoku+dvgBcXm4AdSRVovlHkr4NTN5NW0uB\nO4H/LP1eQpX8HgusKEOqrwSuHezvERERMVT5lzcRERERERFRW7nSGhEREREREbWVpDUiIiIiIiJq\nK0lrRERERERE1FaS1oiIiIiIiKitJK0RERERERFRW0laIyIiIiIioraStEZERERERERtJWmNiIiI\niIiI2vo/AlzJeFBmoLYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b4db254128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mask = select.get_support()\n",
    "plt.matshow(mask.reshape(1,-1), cmap='gray_r')\n",
    "plt.xlabel('Index of Features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The score of Logistic Regression with the selected features on the test set: 0.951\n"
     ]
    }
   ],
   "source": [
    "X_test_s = select.transform(X_test)\n",
    "score = LogisticRegression().fit(X_train_s, y_train).score(X_test_s, y_test)\n",
    "print('The score of Logistic Regression with the selected features on the test set: {:.3f}'.format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}