Step 2 Identify Features UCI Madelon Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn import tree\n",
    "from sklearn import svm\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import linear_model\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import SelectKBest, SelectFromModel, f_regression\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing and sampling the UCI data / basic EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_data = pd.read_csv('madelon_train.data.csv', delimiter=' ', header=None).drop(500, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_labels = pd.read_csv('madelon_train.labels.csv', delimiter=' ', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_10pct_1 = df_train_data.sample(200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df_train_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_10pct_1 = y.sample(200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>490</th>\n",
       "      <th>491</th>\n",
       "      <th>492</th>\n",
       "      <th>493</th>\n",
       "      <th>494</th>\n",
       "      <th>495</th>\n",
       "      <th>496</th>\n",
       "      <th>497</th>\n",
       "      <th>498</th>\n",
       "      <th>499</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>485</td>\n",
       "      <td>477</td>\n",
       "      <td>537</td>\n",
       "      <td>479</td>\n",
       "      <td>452</td>\n",
       "      <td>471</td>\n",
       "      <td>491</td>\n",
       "      <td>476</td>\n",
       "      <td>475</td>\n",
       "      <td>473</td>\n",
       "      <td>...</td>\n",
       "      <td>477</td>\n",
       "      <td>481</td>\n",
       "      <td>477</td>\n",
       "      <td>485</td>\n",
       "      <td>511</td>\n",
       "      <td>485</td>\n",
       "      <td>481</td>\n",
       "      <td>479</td>\n",
       "      <td>475</td>\n",
       "      <td>496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>483</td>\n",
       "      <td>458</td>\n",
       "      <td>460</td>\n",
       "      <td>487</td>\n",
       "      <td>587</td>\n",
       "      <td>475</td>\n",
       "      <td>526</td>\n",
       "      <td>479</td>\n",
       "      <td>485</td>\n",
       "      <td>469</td>\n",
       "      <td>...</td>\n",
       "      <td>463</td>\n",
       "      <td>478</td>\n",
       "      <td>487</td>\n",
       "      <td>338</td>\n",
       "      <td>513</td>\n",
       "      <td>486</td>\n",
       "      <td>483</td>\n",
       "      <td>492</td>\n",
       "      <td>510</td>\n",
       "      <td>517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>487</td>\n",
       "      <td>542</td>\n",
       "      <td>499</td>\n",
       "      <td>468</td>\n",
       "      <td>448</td>\n",
       "      <td>471</td>\n",
       "      <td>442</td>\n",
       "      <td>478</td>\n",
       "      <td>480</td>\n",
       "      <td>477</td>\n",
       "      <td>...</td>\n",
       "      <td>487</td>\n",
       "      <td>481</td>\n",
       "      <td>492</td>\n",
       "      <td>650</td>\n",
       "      <td>506</td>\n",
       "      <td>501</td>\n",
       "      <td>480</td>\n",
       "      <td>489</td>\n",
       "      <td>499</td>\n",
       "      <td>498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>480</td>\n",
       "      <td>491</td>\n",
       "      <td>510</td>\n",
       "      <td>485</td>\n",
       "      <td>495</td>\n",
       "      <td>472</td>\n",
       "      <td>417</td>\n",
       "      <td>474</td>\n",
       "      <td>502</td>\n",
       "      <td>476</td>\n",
       "      <td>...</td>\n",
       "      <td>491</td>\n",
       "      <td>480</td>\n",
       "      <td>474</td>\n",
       "      <td>572</td>\n",
       "      <td>454</td>\n",
       "      <td>469</td>\n",
       "      <td>475</td>\n",
       "      <td>482</td>\n",
       "      <td>494</td>\n",
       "      <td>461</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 500 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0    1    2    3    4    5    6    7    8    9   ...   490  491  492  493  \\\n",
       "0  485  477  537  479  452  471  491  476  475  473 ...   477  481  477  485   \n",
       "1  483  458  460  487  587  475  526  479  485  469 ...   463  478  487  338   \n",
       "2  487  542  499  468  448  471  442  478  480  477 ...   487  481  492  650   \n",
       "3  480  491  510  485  495  472  417  474  502  476 ...   491  480  474  572   \n",
       "\n",
       "   494  495  496  497  498  499  \n",
       "0  511  485  481  479  475  496  \n",
       "1  513  486  483  492  510  517  \n",
       "2  506  501  480  489  499  498  \n",
       "3  454  469  475  482  494  461  \n",
       "\n",
       "[4 rows x 500 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_data.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "525    -1\n",
       "485     1\n",
       "1735   -1\n",
       "199     1\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_10pct_1.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((200, 500), (200,))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data_10pct_1.shape, y_10pct_1.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SELECTKBEST for Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Used Y Labels instead of dropping features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(train_data_10pct_1,\n",
    "                                                    y_10pct_1,\n",
    "                                                    test_size = .3,\n",
    "                                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SelectKBest(k=20, score_func=<function f_classif at 0x7f93086c3400>)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skb = SelectKBest(k=20)\n",
    "\n",
    "skb.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 47, 100, 115, 173, 183, 189, 211, 234, 245, 249, 254, 279, 286,\n",
       "       303, 345, 349, 370, 371, 400, 416])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skb_feats = np.where(skb.get_support())[0]\n",
    "skb_feats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SELECT FROM MODEL for Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Used Y Labels instead of dropping features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "sfm = SelectFromModel(LogisticRegression(), threshold='mean')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SelectFromModel(estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "        prefit=False, threshold='mean')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfm.fit(X_train_scaled, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1,   4,   8,  10,  11,  12,  13,  18,  21,  23,  28,  30,  31,\n",
       "        32,  33,  34,  41,  42,  43,  45,  47,  48,  49,  51,  52,  53,\n",
       "        54,  57,  58,  59,  60,  61,  63,  68,  69,  75,  78,  79,  83,\n",
       "        85,  92,  96, 100, 103, 104, 107, 108, 109, 111, 113, 114, 115,\n",
       "       116, 118, 121, 122, 123, 125, 127, 129, 131, 140, 142, 143, 146,\n",
       "       147, 148, 155, 156, 157, 159, 163, 167, 168, 169, 171, 173, 175,\n",
       "       176, 177, 181, 182, 183, 185, 186, 187, 189, 190, 192, 193, 197,\n",
       "       199, 200, 201, 202, 203, 205, 206, 211, 215, 219, 220, 221, 223,\n",
       "       224, 225, 226, 234, 236, 240, 241, 242, 243, 245, 246, 249, 251,\n",
       "       254, 256, 257, 258, 262, 264, 268, 269, 270, 272, 273, 277, 278,\n",
       "       279, 280, 284, 286, 287, 293, 294, 295, 296, 298, 299, 303, 304,\n",
       "       308, 309, 311, 312, 313, 315, 317, 318, 324, 327, 328, 333, 334,\n",
       "       337, 342, 344, 345, 349, 351, 352, 355, 356, 357, 358, 363, 368,\n",
       "       369, 370, 371, 372, 374, 378, 379, 384, 389, 391, 395, 399, 400,\n",
       "       402, 403, 404, 410, 413, 415, 416, 417, 422, 423, 425, 427, 437,\n",
       "       440, 444, 446, 451, 456, 458, 459, 462, 464, 470, 471, 473, 474,\n",
       "       475, 476, 478, 482, 483, 484, 485, 487, 497, 498])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sfm_feats = np.where(sfm.get_support())[0]\n",
    "sfm_feats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: All the features in the SelectFromModel technique exist in SelectKBest. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PEARSON CORRELATION MASKING For Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_df = df_train_data.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "corrs = list(corr_df[corr_df[corr_df.abs() >.5].count() > 1].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[28,\n",
       " 48,\n",
       " 64,\n",
       " 105,\n",
       " 128,\n",
       " 153,\n",
       " 241,\n",
       " 281,\n",
       " 318,\n",
       " 336,\n",
       " 338,\n",
       " 378,\n",
       " 433,\n",
       " 442,\n",
       " 451,\n",
       " 453,\n",
       " 455,\n",
       " 472,\n",
       " 475,\n",
       " 493]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp_feat_df = df_train_data.iloc[:, corrs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>28</th>\n",
       "      <th>48</th>\n",
       "      <th>64</th>\n",
       "      <th>105</th>\n",
       "      <th>128</th>\n",
       "      <th>153</th>\n",
       "      <th>241</th>\n",
       "      <th>281</th>\n",
       "      <th>318</th>\n",
       "      <th>336</th>\n",
       "      <th>338</th>\n",
       "      <th>378</th>\n",
       "      <th>433</th>\n",
       "      <th>442</th>\n",
       "      <th>451</th>\n",
       "      <th>453</th>\n",
       "      <th>455</th>\n",
       "      <th>472</th>\n",
       "      <th>475</th>\n",
       "      <th>493</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>459</td>\n",
       "      <td>440</td>\n",
       "      <td>648</td>\n",
       "      <td>181</td>\n",
       "      <td>452</td>\n",
       "      <td>575</td>\n",
       "      <td>434</td>\n",
       "      <td>517</td>\n",
       "      <td>414</td>\n",
       "      <td>658</td>\n",
       "      <td>628</td>\n",
       "      <td>419</td>\n",
       "      <td>533</td>\n",
       "      <td>568</td>\n",
       "      <td>463</td>\n",
       "      <td>471</td>\n",
       "      <td>630</td>\n",
       "      <td>515</td>\n",
       "      <td>401</td>\n",
       "      <td>485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>475</td>\n",
       "      <td>499</td>\n",
       "      <td>488</td>\n",
       "      <td>431</td>\n",
       "      <td>473</td>\n",
       "      <td>404</td>\n",
       "      <td>551</td>\n",
       "      <td>435</td>\n",
       "      <td>469</td>\n",
       "      <td>469</td>\n",
       "      <td>528</td>\n",
       "      <td>526</td>\n",
       "      <td>442</td>\n",
       "      <td>463</td>\n",
       "      <td>474</td>\n",
       "      <td>311</td>\n",
       "      <td>582</td>\n",
       "      <td>465</td>\n",
       "      <td>549</td>\n",
       "      <td>338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>491</td>\n",
       "      <td>460</td>\n",
       "      <td>485</td>\n",
       "      <td>593</td>\n",
       "      <td>487</td>\n",
       "      <td>585</td>\n",
       "      <td>474</td>\n",
       "      <td>535</td>\n",
       "      <td>506</td>\n",
       "      <td>465</td>\n",
       "      <td>431</td>\n",
       "      <td>464</td>\n",
       "      <td>569</td>\n",
       "      <td>503</td>\n",
       "      <td>481</td>\n",
       "      <td>606</td>\n",
       "      <td>424</td>\n",
       "      <td>485</td>\n",
       "      <td>454</td>\n",
       "      <td>650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>472</td>\n",
       "      <td>529</td>\n",
       "      <td>415</td>\n",
       "      <td>698</td>\n",
       "      <td>493</td>\n",
       "      <td>591</td>\n",
       "      <td>569</td>\n",
       "      <td>526</td>\n",
       "      <td>458</td>\n",
       "      <td>398</td>\n",
       "      <td>377</td>\n",
       "      <td>553</td>\n",
       "      <td>565</td>\n",
       "      <td>447</td>\n",
       "      <td>472</td>\n",
       "      <td>545</td>\n",
       "      <td>456</td>\n",
       "      <td>457</td>\n",
       "      <td>602</td>\n",
       "      <td>572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>472</td>\n",
       "      <td>429</td>\n",
       "      <td>387</td>\n",
       "      <td>451</td>\n",
       "      <td>475</td>\n",
       "      <td>448</td>\n",
       "      <td>538</td>\n",
       "      <td>456</td>\n",
       "      <td>462</td>\n",
       "      <td>385</td>\n",
       "      <td>509</td>\n",
       "      <td>424</td>\n",
       "      <td>462</td>\n",
       "      <td>536</td>\n",
       "      <td>472</td>\n",
       "      <td>426</td>\n",
       "      <td>465</td>\n",
       "      <td>500</td>\n",
       "      <td>560</td>\n",
       "      <td>435</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   28   48   64   105  128  153  241  281  318  336  338  378  433  442  451  \\\n",
       "0  459  440  648  181  452  575  434  517  414  658  628  419  533  568  463   \n",
       "1  475  499  488  431  473  404  551  435  469  469  528  526  442  463  474   \n",
       "2  491  460  485  593  487  585  474  535  506  465  431  464  569  503  481   \n",
       "3  472  529  415  698  493  591  569  526  458  398  377  553  565  447  472   \n",
       "4  472  429  387  451  475  448  538  456  462  385  509  424  462  536  472   \n",
       "\n",
       "   453  455  472  475  493  \n",
       "0  471  630  515  401  485  \n",
       "1  311  582  465  549  338  \n",
       "2  606  424  485  454  650  \n",
       "3  545  456  457  602  572  \n",
       "4  426  465  500  560  435  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp_feat_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## JOSH'S METHOD WITH KNN For Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_r_2_for_feature_with_KNN(data, feature):\n",
    "    new_data = data.drop(feature, axis=1)\n",
    "\n",
    "    X_train, \\\n",
    "    X_test,  \\\n",
    "    y_train, \\\n",
    "    y_test = train_test_split(\n",
    "        new_data,data[feature],test_size=0.25\n",
    "    )\n",
    "    \n",
    "#     scaler = StandardScaler()\n",
    "#     scaler.fit(X_train, y_train)\n",
    "#     X_tr_sc = scaler.transform(X_train, y_train)\n",
    "#     X_ts_sc = scaler.transform(X_test, y_test)\n",
    "\n",
    "    regressor = KNeighborsRegressor()\n",
    "    regressor.fit(X_train, y_train)\n",
    "\n",
    "    score = regressor.score(X_test, y_test)\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_r2_for_feature_knn(data, feature):\n",
    "    scores = []\n",
    "    for _ in range(100):\n",
    "        scores.append(calculate_r_2_for_feature_with_KNN(data, feature))\n",
    "        \n",
    "    scores = np.array(scores)\n",
    "    return scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -0.245046979295\n",
      "1 -0.238925676168\n",
      "2 -0.195839297271\n",
      "3 -0.207651775921\n",
      "4 -0.290965546836\n",
      "5 -0.186031961279\n",
      "6 -0.216777881343\n",
      "7 -0.22852013329\n",
      "8 -0.221416913619\n",
      "9 -0.22328209138\n",
      "10 -0.318268885236\n",
      "11 -0.219157795786\n",
      "12 -0.178603747929\n",
      "13 -0.245168102209\n",
      "14 -0.239491599692\n",
      "15 -0.256422742041\n",
      "16 -0.293247965137\n",
      "17 -0.289946144582\n",
      "18 -0.270950427331\n",
      "19 -0.209296180575\n",
      "20 -0.242068306785\n",
      "21 -0.361302534394\n",
      "22 -0.252615732\n",
      "23 -0.168248916448\n",
      "24 -0.288563021175\n",
      "25 -0.177480884715\n",
      "26 -0.237122757349\n",
      "27 -0.256043158178\n",
      "28 0.496747259215\n",
      "29 -0.2271373858\n",
      "30 -0.234383160187\n",
      "31 -0.323866301254\n",
      "32 -0.22766524785\n",
      "33 -0.243838775844\n",
      "34 -0.177611404275\n",
      "35 -0.197676629037\n",
      "36 -0.393927081165\n",
      "37 -0.289477672842\n",
      "38 -0.256457505459\n",
      "39 -0.202866432586\n",
      "40 -0.174622036957\n",
      "41 -0.229961633427\n",
      "42 -0.231382713907\n",
      "43 -0.292839034093\n",
      "44 -0.135473120734\n",
      "45 -0.260518894539\n",
      "46 -0.271942387684\n",
      "47 -0.201314072851\n",
      "48 0.34547806554\n",
      "49 -0.216840391025\n",
      "50 -0.272817121185\n",
      "51 -0.286391120442\n",
      "52 -0.263805116557\n",
      "53 -0.22970983666\n",
      "54 -0.25709097108\n",
      "55 -0.338608351933\n",
      "56 -0.166972012266\n",
      "57 -0.301822465342\n",
      "58 -0.154850435204\n",
      "59 -0.168411959744\n",
      "60 -0.218055718464\n",
      "61 -0.226210547538\n",
      "62 -0.262532620539\n",
      "63 -0.229177528782\n",
      "64 0.77221490597\n",
      "65 -0.325092446449\n",
      "66 -0.193680825974\n",
      "67 -0.227475868355\n",
      "68 -0.167059550824\n",
      "69 -0.217449566465\n",
      "70 -0.255102333234\n",
      "71 -0.198079387558\n",
      "72 -0.267952961182\n",
      "73 -0.389878655307\n",
      "74 -0.176956723024\n",
      "75 -0.314787620975\n",
      "76 -0.304261735296\n",
      "77 -0.208334233864\n",
      "78 -0.302948699244\n",
      "79 -0.170305693154\n",
      "80 -0.262464790372\n",
      "81 -0.321661770671\n",
      "82 -0.278501091694\n",
      "83 -0.127534017071\n",
      "84 -0.21511764727\n",
      "85 -0.260973274628\n",
      "86 -0.233608906887\n",
      "87 -0.19553051209\n",
      "88 -0.214363112689\n",
      "89 -0.346016450327\n",
      "90 -0.0830034268112\n",
      "91 -0.217081724767\n",
      "92 -0.233928614056\n",
      "93 -0.277882134511\n",
      "94 -0.152677577144\n",
      "95 -0.258579530879\n",
      "96 -0.341738613048\n",
      "97 -0.151824901215\n",
      "98 -0.310153468405\n",
      "99 -0.126991755358\n",
      "100 -0.279971827586\n",
      "101 -0.269923652623\n",
      "102 -0.265012780089\n",
      "103 -0.24673685351\n",
      "104 -0.187438390856\n",
      "105 0.685503500549\n",
      "106 -0.271481530358\n",
      "107 -0.371474506667\n",
      "108 -0.35990317792\n",
      "109 -0.218331370027\n",
      "110 -0.217055602674\n",
      "111 -0.222754264932\n",
      "112 -0.137386152315\n",
      "113 -0.25096233098\n",
      "114 -0.199970104499\n",
      "115 -0.190363167869\n",
      "116 -0.178431674082\n",
      "117 -0.366312876159\n",
      "118 -0.234904598039\n",
      "119 -0.148221590329\n",
      "120 -0.218590272571\n",
      "121 -0.151973801577\n",
      "122 -0.188114938534\n",
      "123 -0.290933809794\n",
      "124 -0.304607102135\n",
      "125 -0.197942851548\n",
      "126 -0.111687502245\n",
      "127 -0.229012579083\n",
      "128 0.815724918936\n",
      "129 -0.155916075684\n",
      "130 -0.0896818433234\n",
      "131 -0.225180172051\n",
      "132 -0.196942242767\n",
      "133 -0.14114869995\n",
      "134 -0.282025384046\n",
      "135 -0.23524083995\n",
      "136 -0.186117082663\n",
      "137 -0.347179379783\n",
      "138 -0.232694024207\n",
      "139 -0.141147021612\n",
      "140 -0.257431496563\n",
      "141 -0.258665956343\n",
      "142 -0.18885992042\n",
      "143 -0.169000752881\n",
      "144 -0.25530945274\n",
      "145 -0.162809984419\n",
      "146 -0.199354770676\n",
      "147 -0.10515045465\n",
      "148 -0.221919898565\n",
      "149 -0.0671563667406\n",
      "150 -0.267370497732\n",
      "151 -0.249678470258\n",
      "152 -0.197599759418\n",
      "153 0.739453340156\n",
      "154 -0.251482723861\n",
      "155 -0.479194773029\n",
      "156 -0.222751574089\n",
      "157 -0.0634096576209\n",
      "158 -0.23662886423\n",
      "159 -0.316129808568\n",
      "160 -0.266621871095\n",
      "161 -0.213921734849\n",
      "162 -0.33133522934\n",
      "163 -0.261925746236\n",
      "164 -0.244551554016\n",
      "165 -0.224395540384\n",
      "166 -0.325256443077\n",
      "167 -0.323426442711\n",
      "168 -0.299007847958\n",
      "169 -0.220662255674\n",
      "170 -0.203871554596\n",
      "171 -0.227848044103\n",
      "172 -0.303199389047\n",
      "173 -0.421431254747\n",
      "174 -0.126674949794\n",
      "175 -0.186626692349\n",
      "176 -0.276134455974\n",
      "177 -0.157824039987\n",
      "178 -0.277032136801\n",
      "179 -0.168176136976\n",
      "180 -0.247281873394\n",
      "181 -0.243770582605\n",
      "182 -0.165711164558\n",
      "183 -0.312617855294\n",
      "184 -0.294242336131\n",
      "185 -0.281207523012\n",
      "186 -0.212317369078\n",
      "187 -0.222410671557\n",
      "188 -0.300036844467\n",
      "189 -0.248865751617\n",
      "190 -0.236002453249\n",
      "191 -0.247494619386\n",
      "192 -0.265191602316\n",
      "193 -0.20331330405\n",
      "194 -0.136419643307\n",
      "195 -0.185571562212\n",
      "196 -0.169107747495\n",
      "197 -0.202547341471\n",
      "198 -0.137543281268\n",
      "199 -0.30884853216\n",
      "200 -0.356894795879\n",
      "201 -0.180999186354\n",
      "202 -0.21439514754\n",
      "203 -0.221494471644\n",
      "204 -0.264613793228\n",
      "205 -0.172271164977\n",
      "206 -0.254769992171\n",
      "207 -0.24556208563\n",
      "208 -0.174125558941\n",
      "209 -0.242692407967\n",
      "210 -0.238184823566\n",
      "211 -0.292281154018\n",
      "212 -0.215337776565\n",
      "213 -0.16464439461\n",
      "214 -0.309601648668\n",
      "215 -0.266320196516\n",
      "216 -0.110487854242\n",
      "217 -0.204959269196\n",
      "218 -0.223868901115\n",
      "219 -0.34742193058\n",
      "220 -0.22669578015\n",
      "221 -0.200840783126\n",
      "222 -0.299424224973\n",
      "223 -0.235471548314\n",
      "224 -0.258319523395\n",
      "225 -0.373257711112\n",
      "226 -0.333869040697\n",
      "227 -0.319334352713\n",
      "228 -0.315383032556\n",
      "229 -0.172768623904\n",
      "230 -0.19413131222\n",
      "231 -0.197514324079\n",
      "232 -0.304574188399\n",
      "233 -0.0345421150943\n",
      "234 -0.240196934142\n",
      "235 -0.253491322536\n",
      "236 -0.283412707761\n",
      "237 -0.19531763873\n",
      "238 -0.275751285501\n",
      "239 -0.226842240762\n",
      "240 -0.306308623705\n",
      "241 0.768504249451\n",
      "242 -0.0282033958448\n",
      "243 -0.125343956136\n",
      "244 -0.0862787566895\n",
      "245 -0.273100914995\n",
      "246 -0.19041177644\n",
      "247 -0.167365994828\n",
      "248 -0.16213941699\n",
      "249 -0.144294554281\n",
      "250 -0.261343630397\n",
      "251 -0.246663541729\n",
      "252 -0.213735570105\n",
      "253 -0.108045155533\n",
      "254 -0.25323309611\n",
      "255 -0.27807957102\n",
      "256 -0.238947766156\n",
      "257 -0.265372614394\n",
      "258 -0.200500600468\n",
      "259 -0.27364729989\n",
      "260 -0.16923543706\n",
      "261 -0.159221993277\n",
      "262 -0.225086360815\n",
      "263 -0.206268291359\n",
      "264 -0.271915843197\n",
      "265 -0.236252069176\n",
      "266 -0.191549299284\n",
      "267 -0.282825927462\n",
      "268 -0.348150791235\n",
      "269 -0.221306441774\n",
      "270 -0.216790777711\n",
      "271 -0.168102208076\n",
      "272 -0.218445785815\n",
      "273 -0.185296655517\n",
      "274 -0.28623439134\n",
      "275 -0.178044788943\n",
      "276 -0.298141610117\n",
      "277 -0.283505242283\n",
      "278 -0.214127061467\n",
      "279 -0.204050517103\n",
      "280 -0.129680113746\n",
      "281 0.811324298967\n",
      "282 -0.274429397115\n",
      "283 -0.245307283486\n",
      "284 -0.308758159097\n",
      "285 -0.236694414645\n",
      "286 -0.296098792995\n",
      "287 -0.174416473244\n",
      "288 -0.265957376403\n",
      "289 -0.201824780962\n",
      "290 -0.279140845804\n",
      "291 -0.307635606095\n",
      "292 -0.205466064206\n",
      "293 -0.251175500388\n",
      "294 -0.274549058217\n",
      "295 -0.382020930425\n",
      "296 -0.290397275306\n",
      "297 -0.211698058086\n",
      "298 -0.162943072623\n",
      "299 -0.346525868794\n",
      "300 -0.146686553401\n",
      "301 -0.320537402694\n",
      "302 -0.271485820328\n",
      "303 -0.284550608068\n",
      "304 -0.253207216136\n",
      "305 -0.278762037381\n",
      "306 -0.357850578388\n",
      "307 -0.155876167408\n",
      "308 -0.247389643261\n",
      "309 -0.274580055048\n",
      "310 -0.274126610851\n",
      "311 -0.385427307162\n",
      "312 -0.181758652321\n",
      "313 -0.163017717569\n",
      "314 -0.0934141653196\n",
      "315 -0.218120745799\n",
      "316 -0.32333600879\n",
      "317 -0.100342646783\n",
      "318 0.40848074275\n",
      "319 -0.205928843203\n",
      "320 -0.277041851799\n",
      "321 -0.202668040337\n",
      "322 -0.252618421043\n",
      "323 -0.16474090667\n",
      "324 -0.174693271981\n",
      "325 -0.292767679372\n",
      "326 -0.243422219439\n",
      "327 -0.156896626266\n",
      "328 -0.261048391836\n",
      "329 -0.185074097501\n",
      "330 -0.321072036343\n",
      "331 -0.347787467231\n",
      "332 -0.151109326168\n",
      "333 -0.303169762236\n",
      "334 -0.190114300519\n",
      "335 -0.152195142793\n",
      "336 0.743747526566\n",
      "337 -0.285121608064\n",
      "338 0.832146384909\n",
      "339 -0.287252824137\n",
      "340 -0.23065361925\n",
      "341 -0.19428493526\n",
      "342 -0.233136469986\n",
      "343 -0.335356582839\n",
      "344 -0.294297066647\n",
      "345 -0.177556364822\n",
      "346 -0.240101052665\n",
      "347 -0.336785586451\n",
      "348 -0.13751005508\n",
      "349 -0.173229267803\n",
      "350 -0.138467211608\n",
      "351 -0.283342809771\n",
      "352 -0.238367357727\n",
      "353 -0.276917950172\n",
      "354 -0.202777121058\n",
      "355 -0.193491331076\n",
      "356 -0.128110165495\n",
      "357 -0.162704743255\n",
      "358 -0.131846948641\n",
      "359 -0.249695954311\n",
      "360 -0.24526438632\n",
      "361 -0.20977499088\n",
      "362 -0.175797020375\n",
      "363 -0.282954991783\n",
      "364 -0.194913555401\n",
      "365 -0.35718634914\n",
      "366 -0.217677687117\n",
      "367 -0.227747966803\n",
      "368 -0.187857852133\n",
      "369 -0.235274972198\n",
      "370 -0.195947357751\n",
      "371 -0.247426014991\n",
      "372 -0.301278622765\n",
      "373 -0.261048910835\n",
      "374 -0.272558138099\n",
      "375 -0.238118888091\n",
      "376 -0.173822030296\n",
      "377 -0.163666434939\n",
      "378 0.232136833779\n",
      "379 -0.238477238191\n",
      "380 -0.332167520537\n",
      "381 -0.278731438843\n",
      "382 -0.13833998042\n",
      "383 -0.15840104795\n",
      "384 -0.207436158747\n",
      "385 -0.171516679792\n",
      "386 -0.261622086009\n",
      "387 -0.214410360424\n",
      "388 -0.229571862383\n",
      "389 -0.222754315298\n",
      "390 -0.134561794549\n",
      "391 -0.200607812744\n",
      "392 -0.214210934237\n",
      "393 -0.125833831185\n",
      "394 -0.206408698805\n",
      "395 -0.138467273558\n",
      "396 -0.30204378979\n",
      "397 -0.318739108423\n",
      "398 -0.183316489501\n",
      "399 -0.208140530044\n",
      "400 -0.192115234442\n",
      "401 -0.185187584672\n",
      "402 -0.180686304284\n",
      "403 -0.266008748303\n",
      "404 -0.160610947311\n",
      "405 -0.188826352789\n",
      "406 -0.315665381\n",
      "407 -0.328704250653\n",
      "408 -0.191374378524\n",
      "409 -0.219419468004\n",
      "410 -0.297124807708\n",
      "411 -0.244297686808\n",
      "412 -0.226927176669\n",
      "413 -0.164842124992\n",
      "414 -0.237151272878\n",
      "415 -0.306786359149\n",
      "416 -0.169101971181\n",
      "417 -0.336442679471\n",
      "418 -0.297631496776\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "419 -0.216614244267\n",
      "420 -0.197812214397\n",
      "421 -0.0726315781183\n",
      "422 -0.302908780955\n",
      "423 -0.230106991028\n",
      "424 -0.246172336024\n",
      "425 -0.204131454895\n",
      "426 -0.304711116094\n",
      "427 -0.287979246557\n",
      "428 -0.205795615942\n",
      "429 -0.318337552124\n",
      "430 -0.166955894222\n",
      "431 -0.240398578975\n",
      "432 -0.261686703119\n",
      "433 0.798098582639\n",
      "434 -0.169287524157\n",
      "435 -0.199091267318\n",
      "436 -0.441972723868\n",
      "437 -0.165549468867\n",
      "438 -0.319340552263\n",
      "439 -0.260873112526\n",
      "440 -0.287983412437\n",
      "441 -0.264810872006\n",
      "442 0.712780509012\n",
      "443 -0.115768470997\n",
      "444 -0.183097393634\n",
      "445 -0.184623986647\n",
      "446 -0.282303665309\n",
      "447 -0.294685903823\n",
      "448 -0.190612301519\n",
      "449 -0.172727257191\n",
      "450 -0.222374199754\n",
      "451 0.499461060497\n",
      "452 -0.141401135303\n",
      "453 0.835594995345\n",
      "454 -0.235244294889\n",
      "455 0.807623712136\n",
      "456 -0.213264400708\n",
      "457 -0.206018246566\n",
      "458 -0.207071577527\n",
      "459 -0.18353907984\n",
      "460 -0.220604481363\n",
      "461 -0.163191493801\n",
      "462 -0.348617201664\n",
      "463 -0.240562510195\n",
      "464 -0.163951298414\n",
      "465 -0.17174593314\n",
      "466 -0.330330649475\n",
      "467 -0.187806612566\n",
      "468 -0.194532830712\n",
      "469 -0.289689688836\n",
      "470 -0.201781811121\n",
      "471 -0.232365377639\n",
      "472 0.801568596005\n",
      "473 -0.234432461305\n",
      "474 -0.183134407324\n",
      "475 0.739923078833\n",
      "476 -0.285560858471\n",
      "477 -0.359695453316\n",
      "478 -0.156059187597\n",
      "479 -0.309986373068\n",
      "480 -0.156525093271\n",
      "481 -0.185115088375\n",
      "482 -0.216602062148\n",
      "483 -0.0757129394243\n",
      "484 -0.15329769919\n",
      "485 -0.145502897211\n",
      "486 -0.284348014345\n",
      "487 -0.284894544422\n",
      "488 -0.0925760465448\n",
      "489 -0.343467060492\n",
      "490 -0.317514209705\n",
      "491 -0.247893764549\n",
      "492 -0.29839363483\n",
      "493 0.831146296372\n",
      "494 -0.223483228835\n",
      "495 -0.214800496846\n",
      "496 -0.0776418774789\n",
      "497 -0.200040413115\n",
      "498 -0.133683226604\n",
      "499 -0.211868692844\n"
     ]
    }
   ],
   "source": [
    "r2_knn_scores = []\n",
    "r2_knn_means = []\n",
    "\n",
    "for column in train_data_10pct_1.columns:\n",
    "#     score_function_calculations = calculate_r_2_for_feature_with_KNN(train_data_10pct_1, column)\n",
    "#     r2_knn_scores.append(score_function_calculations)\n",
    "    score_function_means = mean_r2_for_feature_knn(train_data_10pct_1, column)\n",
    "    r2_knn_means.append((column, score_function_means))\n",
    "    print(column, score_function_means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28:  0.4970997525375522\n",
      "48:  0.35864011297371406\n",
      "64:  0.7673534545982738\n",
      "105:  0.6884552637255332\n",
      "128:  0.822643186968861\n",
      "153:  0.7414818390725066\n",
      "241:  0.7657560974466524\n",
      "281:  0.807669537504913\n",
      "318:  0.4280255542212325\n",
      "336:  0.7471183291068563\n",
      "338:  0.8282715333617031\n",
      "378:  0.2552889342339038\n",
      "433:  0.8043099619520632\n",
      "442:  0.7048083922251225\n",
      "451:  0.49965241847853037\n",
      "453:  0.8429839060805597\n",
      "455:  0.8106141167086496\n",
      "472:  0.8001661139014282\n",
      "475:  0.7463076827738361\n",
      "493:  0.8316823985811691\n"
     ]
    }
   ],
   "source": [
    "print(\"{} {}\".format(\"28: \", mean_r2_for_feature_knn(train_data_10pct_1, 28)))\n",
    "print(\"{} {}\".format(\"48: \", mean_r2_for_feature_knn(train_data_10pct_1, 48)))\n",
    "print(\"{} {}\".format(\"64: \", mean_r2_for_feature_knn(train_data_10pct_1, 64)))\n",
    "print(\"{} {}\".format(\"105: \", mean_r2_for_feature_knn(train_data_10pct_1, 105)))\n",
    "print(\"{} {}\".format(\"128: \", mean_r2_for_feature_knn(train_data_10pct_1, 128)))\n",
    "print(\"{} {}\".format(\"153: \", mean_r2_for_feature_knn(train_data_10pct_1, 153)))\n",
    "print(\"{} {}\".format(\"241: \", mean_r2_for_feature_knn(train_data_10pct_1, 241)))\n",
    "print(\"{} {}\".format(\"281: \", mean_r2_for_feature_knn(train_data_10pct_1, 281)))\n",
    "print(\"{} {}\".format(\"318: \", mean_r2_for_feature_knn(train_data_10pct_1, 318)))\n",
    "print(\"{} {}\".format(\"336: \", mean_r2_for_feature_knn(train_data_10pct_1, 336)))\n",
    "print(\"{} {}\".format(\"338: \", mean_r2_for_feature_knn(train_data_10pct_1, 338)))\n",
    "print(\"{} {}\".format(\"378: \", mean_r2_for_feature_knn(train_data_10pct_1, 378)))\n",
    "print(\"{} {}\".format(\"433: \", mean_r2_for_feature_knn(train_data_10pct_1, 433)))\n",
    "print(\"{} {}\".format(\"442: \", mean_r2_for_feature_knn(train_data_10pct_1, 442)))\n",
    "print(\"{} {}\".format(\"451: \", mean_r2_for_feature_knn(train_data_10pct_1, 451)))\n",
    "print(\"{} {}\".format(\"453: \", mean_r2_for_feature_knn(train_data_10pct_1, 453)))\n",
    "print(\"{} {}\".format(\"455: \", mean_r2_for_feature_knn(train_data_10pct_1, 455)))\n",
    "print(\"{} {}\".format(\"472: \", mean_r2_for_feature_knn(train_data_10pct_1, 472)))\n",
    "print(\"{} {}\".format(\"475: \", mean_r2_for_feature_knn(train_data_10pct_1, 475)))\n",
    "print(\"{} {}\".format(\"493: \", mean_r2_for_feature_knn(train_data_10pct_1, 493)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "The highest r2 scores returned using Josh's method corroborate with the Pearson Correlation Masking technique results. Both produce the same informative/redundant features. Other methods used such as SelectKBest and SelectFromModel did not produce the same features and were reliant on the labels in both the y_train and y_test, which makes them less reliable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}