code/08_modeling_TDs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "80a017bb-cfa1-43c3-9af1-f38505273e92",
   "metadata": {},
   "source": [
    "## Modeling Touchdowns\n",
    "\n",
    "This notebook builds out a model to predict touchdowns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f942e860-d1ed-4414-864c-844709c96ed5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n",
    "from sklearn.linear_model import LinearRegression, Ridge, RidgeCV, Lasso, LassoCV, ElasticNet, ElasticNetCV\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor \n",
    "from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor, BaggingRegressor, StackingRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn import metrics\n",
    "\n",
    "np.random.seed(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "47cf873e-fd94-499a-8618-6385f43ef060",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tm</th>\n",
       "      <th>Age</th>\n",
       "      <th>G</th>\n",
       "      <th>GS</th>\n",
       "      <th>Tgt</th>\n",
       "      <th>Rec</th>\n",
       "      <th>Yds</th>\n",
       "      <th>TD</th>\n",
       "      <th>1D</th>\n",
       "      <th>YBC</th>\n",
       "      <th>...</th>\n",
       "      <th>Yds_-2_year</th>\n",
       "      <th>TD_-2_year</th>\n",
       "      <th>Rnd</th>\n",
       "      <th>Pick</th>\n",
       "      <th>Pos</th>\n",
       "      <th>YrsPlayed</th>\n",
       "      <th>Tgt_target</th>\n",
       "      <th>Rec_target</th>\n",
       "      <th>Yds_target</th>\n",
       "      <th>TD_target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Player</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Stefon Diggs</th>\n",
       "      <td>BUF</td>\n",
       "      <td>27</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>166</td>\n",
       "      <td>127</td>\n",
       "      <td>1535</td>\n",
       "      <td>8</td>\n",
       "      <td>73.0</td>\n",
       "      <td>1071</td>\n",
       "      <td>...</td>\n",
       "      <td>1021.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>WR</td>\n",
       "      <td>5.0</td>\n",
       "      <td>164</td>\n",
       "      <td>103</td>\n",
       "      <td>1225</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Davante Adams</th>\n",
       "      <td>GNB</td>\n",
       "      <td>28</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>149</td>\n",
       "      <td>115</td>\n",
       "      <td>1374</td>\n",
       "      <td>18</td>\n",
       "      <td>73.0</td>\n",
       "      <td>777</td>\n",
       "      <td>...</td>\n",
       "      <td>1386.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>WR</td>\n",
       "      <td>6.0</td>\n",
       "      <td>169</td>\n",
       "      <td>123</td>\n",
       "      <td>1553</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeAndre Hopkins</th>\n",
       "      <td>ARI</td>\n",
       "      <td>28</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>160</td>\n",
       "      <td>115</td>\n",
       "      <td>1407</td>\n",
       "      <td>6</td>\n",
       "      <td>75.0</td>\n",
       "      <td>873</td>\n",
       "      <td>...</td>\n",
       "      <td>1572.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>WR</td>\n",
       "      <td>7.0</td>\n",
       "      <td>64</td>\n",
       "      <td>42</td>\n",
       "      <td>572</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Darren Waller</th>\n",
       "      <td>LVR</td>\n",
       "      <td>28</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>145</td>\n",
       "      <td>107</td>\n",
       "      <td>1196</td>\n",
       "      <td>9</td>\n",
       "      <td>69.0</td>\n",
       "      <td>624</td>\n",
       "      <td>...</td>\n",
       "      <td>75.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>TE</td>\n",
       "      <td>5.0</td>\n",
       "      <td>93</td>\n",
       "      <td>55</td>\n",
       "      <td>665</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Travis Kelce</th>\n",
       "      <td>KAN</td>\n",
       "      <td>31</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>145</td>\n",
       "      <td>105</td>\n",
       "      <td>1416</td>\n",
       "      <td>11</td>\n",
       "      <td>79.0</td>\n",
       "      <td>829</td>\n",
       "      <td>...</td>\n",
       "      <td>1336.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>TE</td>\n",
       "      <td>7.0</td>\n",
       "      <td>134</td>\n",
       "      <td>92</td>\n",
       "      <td>1125</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 44 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Tm  Age   G  GS  Tgt  Rec   Yds  TD    1D   YBC  ...  \\\n",
       "Player                                                             ...   \n",
       "Stefon Diggs     BUF   27  16  15  166  127  1535   8  73.0  1071  ...   \n",
       "Davante Adams    GNB   28  14  14  149  115  1374  18  73.0   777  ...   \n",
       "DeAndre Hopkins  ARI   28  16  16  160  115  1407   6  75.0   873  ...   \n",
       "Darren Waller    LVR   28  16  15  145  107  1196   9  69.0   624  ...   \n",
       "Travis Kelce     KAN   31  15  15  145  105  1416  11  79.0   829  ...   \n",
       "\n",
       "                 Yds_-2_year  TD_-2_year  Rnd   Pick  Pos  YrsPlayed  \\\n",
       "Player                                                                 \n",
       "Stefon Diggs          1021.0         9.0  5.0  146.0   WR        5.0   \n",
       "Davante Adams         1386.0        13.0  2.0   53.0   WR        6.0   \n",
       "DeAndre Hopkins       1572.0        11.0  1.0   27.0   WR        7.0   \n",
       "Darren Waller           75.0         0.0  6.0  204.0   TE        5.0   \n",
       "Travis Kelce          1336.0        10.0  3.0   63.0   TE        7.0   \n",
       "\n",
       "                 Tgt_target  Rec_target  Yds_target  TD_target  \n",
       "Player                                                          \n",
       "Stefon Diggs            164         103        1225         10  \n",
       "Davante Adams           169         123        1553         11  \n",
       "DeAndre Hopkins          64          42         572          8  \n",
       "Darren Waller            93          55         665          2  \n",
       "Travis Kelce            134          92        1125          9  \n",
       "\n",
       "[5 rows x 44 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('../data/receiving_train.csv', index_col = 'Player')\n",
    "df.dropna(inplace = True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c488a86-712a-4f70-a27c-5639a479f567",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(985, 44)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c7b5cc7-4aa3-4c4a-8b1d-7699febd6d05",
   "metadata": {},
   "source": [
    "### Baseline Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1c336b39-1422-4e2f-96d3-926a46820210",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Tgt_target     43.713706\n",
       "Rec_target     29.422335\n",
       "Yds_target    329.638579\n",
       "TD_target       2.082234\n",
       "dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Tgt_target', 'Rec_target', 'Yds_target', 'TD_target']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8484d256-206a-44ea-9a86-67fae159ed9b",
   "metadata": {},
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "925f09e6-3f1f-49af-9ddd-fab01efaebaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(columns = ['Tm', 'Pos', 'Player-additional', 'Tgt_target', 'Rec_target', 'Yds_target', 'TD_target', 'Year'])\n",
    "y = df['TD_target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4d7a6ee5-a7f9-4ce2-9e42-6ccc0cc25030",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "53b5f389-b335-4845-80f8-41f5a15dd715",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.4581210383244573\n",
      "Testing Score: 0.301512899744226\n"
     ]
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "\n",
    "lr.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {lr.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {lr.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7bf72034-2e21-4614-ad0a-232a02b223bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.5124754560977998\n",
      "Testing Score: 0.23596058941189824\n"
     ]
    }
   ],
   "source": [
    "knn = KNeighborsRegressor()\n",
    "\n",
    "knn.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {knn.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {knn.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8efc6ea4-4713-45d9-b4e4-924381c7686d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 1.0\n",
      "Testing Score: -0.08627617109366126\n"
     ]
    }
   ],
   "source": [
    "dt = DecisionTreeRegressor()\n",
    "\n",
    "dt.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {dt.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {dt.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4d9c9790-b096-4549-af38-4e7852fde42c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.9045581781233969\n",
      "Testing Score: 0.3238903844978033\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestRegressor()\n",
    "\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {rf.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {rf.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3b6b0fa8-8dbd-43d6-af66-183b7d3b030d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.7831253230297777\n",
      "Testing Score: 0.27092983412098604\n"
     ]
    }
   ],
   "source": [
    "gr = GradientBoostingRegressor()\n",
    "\n",
    "gr.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {gr.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {gr.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a2729d4a-e56f-4e98-bffc-de6edbd025bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.4474258327731373\n",
      "Testing Score: 0.147624057544814\n"
     ]
    }
   ],
   "source": [
    "ada = AdaBoostRegressor()\n",
    "\n",
    "ada.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {ada.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {ada.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8e52bc2-5f33-4fb1-8970-2f93a469a356",
   "metadata": {},
   "source": [
    "## Gridsearching\n",
    "\n",
    "I want to add some Gridsearching to see if we can improve any of our models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d301f780-480d-4d87-9065-2657d9722a7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3458643489322286\n",
      "{'max_depth': 3, 'n_estimators': 150}\n",
      "Training Score: 0.520098700735018\n",
      "Testing Score: 0.347092019281106\n"
     ]
    }
   ],
   "source": [
    "rf2 = RandomForestRegressor()\n",
    "\n",
    "rf_params = {\n",
    "    'n_estimators': [100, 150, 200],\n",
    "    'max_depth': [None, 1, 2, 3, 4, 5]\n",
    "}\n",
    "gs = GridSearchCV(rf2, param_grid = rf_params, cv = 5)\n",
    "gs.fit(X_train, y_train)\n",
    "print(gs.best_score_)\n",
    "print(gs.best_params_)\n",
    "\n",
    "print(f'Training Score: {gs.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {gs.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a634496b-cec9-4a3f-bd18-0e9f6d92bfc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3222817225740434\n",
      "{'learning_rate': 0.05, 'max_depth': 1, 'n_estimators': 100}\n",
      "Training Score: 0.4706814819451586\n",
      "Testing Score: 0.32943414771329693\n"
     ]
    }
   ],
   "source": [
    "gb2 = GradientBoostingRegressor()\n",
    "\n",
    "gb_params = {\n",
    "    'learning_rate': [.1, .05, .2],\n",
    "    'n_estimators': [100, 150, 200],\n",
    "    'max_depth': [None, 1, 2, 3, 4, 5]\n",
    "}\n",
    "gs2 = GridSearchCV(gb2, param_grid = gb_params, cv = 5)\n",
    "gs2.fit(X_train, y_train)\n",
    "print(gs2.best_score_)\n",
    "print(gs2.best_params_)\n",
    "\n",
    "print(f'Training Score: {gs2.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {gs2.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "63e1d794-d186-4ea7-9da5-4a33adee9ba7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3438288772977413\n",
      "{'learning_rate': 0.05, 'n_estimators': 100}\n",
      "Training Score: 0.5377519693042514\n",
      "Testing Score: 0.315872896156945\n"
     ]
    }
   ],
   "source": [
    "ada2 = AdaBoostRegressor()\n",
    "\n",
    "ada_params = {\n",
    "    'learning_rate': [.1, .05, .2],\n",
    "    'n_estimators': [100, 150, 200],\n",
    "}\n",
    "gs3 = GridSearchCV(ada2, param_grid = ada_params, cv = 5)\n",
    "gs3.fit(X_train, y_train)\n",
    "print(gs3.best_score_)\n",
    "print(gs3.best_params_)\n",
    "\n",
    "print(f'Training Score: {gs3.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {gs3.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f1ec3aa-e73c-4a06-96b7-9856db8d9976",
   "metadata": {},
   "source": [
    "All 3 of our models look better after running the Gridsearch, but the RandomForest still looks the best. It is a little overfit, but all 3 of our models are overfit."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acf5b110-bbdb-4884-a39f-4d5bb0de1e94",
   "metadata": {},
   "source": [
    "## Ridge\n",
    "\n",
    "Next, I want to add regularization using a Ridge model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "81e6dcee-e398-45c9-9478-745afb1c39ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(columns = ['Tm', 'Pos', 'Player-additional', 'Tgt_target', 'Rec_target', 'Yds_target', 'TD_target', 'Year'])\n",
    "y = df['TD_target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d1e66de1-29f3-43cb-8b3e-77497c258856",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3d08ae76-8d3a-4d77-b88b-a787e950117e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train_sc = ss.fit_transform(X_train)\n",
    "X_test_sc = ss.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbbaa06e-d9dc-4d36-9b8e-d1b281ccf07c",
   "metadata": {},
   "source": [
    "Ridge with Default alpha = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d103b833-7e3d-488b-841f-0f90bda274e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.45797854506085467\n",
      "Testing Score: 0.3042513527776862\n"
     ]
    }
   ],
   "source": [
    "ridge = Ridge()\n",
    "\n",
    "ridge.fit(X_train_sc, y_train)\n",
    "\n",
    "print(f'Training Score: {ridge.score(X_train_sc, y_train)}')\n",
    "print(f'Testing Score: {ridge.score(X_test_sc, y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97c55392-9109-4a78-981a-1b0dc0981af4",
   "metadata": {},
   "source": [
    "Find the best alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b8e988e8-eca3-4212-89c4-474cd8cc4efa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RidgeCV(alphas=array([1.00000000e+00, 1.12332403e+00, 1.26185688e+00, 1.41747416e+00,\n",
       "       1.59228279e+00, 1.78864953e+00, 2.00923300e+00, 2.25701972e+00,\n",
       "       2.53536449e+00, 2.84803587e+00, 3.19926714e+00, 3.59381366e+00,\n",
       "       4.03701726e+00, 4.53487851e+00, 5.09413801e+00, 5.72236766e+00,\n",
       "       6.42807312e+00, 7.22080902e+00, 8.11130831e+00, 9.11162756e+00,\n",
       "       1.02353102e+01, 1.14975700e+0...\n",
       "       6.89261210e+03, 7.74263683e+03, 8.69749003e+03, 9.77009957e+03,\n",
       "       1.09749877e+04, 1.23284674e+04, 1.38488637e+04, 1.55567614e+04,\n",
       "       1.74752840e+04, 1.96304065e+04, 2.20513074e+04, 2.47707636e+04,\n",
       "       2.78255940e+04, 3.12571585e+04, 3.51119173e+04, 3.94420606e+04,\n",
       "       4.43062146e+04, 4.97702356e+04, 5.59081018e+04, 6.28029144e+04,\n",
       "       7.05480231e+04, 7.92482898e+04, 8.90215085e+04, 1.00000000e+05]),\n",
       "        cv=5)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphas = np.logspace(0, 5, 100)\n",
    "ridge_cv = RidgeCV(alphas = alphas, cv = 5)\n",
    "ridge_cv.fit(X_train_sc, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2f655908-99d1-4b62-854e-747b2c6afbf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "335.1602650938841"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ridge_cv.alpha_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3c38e2ac-1260-4b07-a3a1-2dce9e310c8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.389360902046714"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ridge_cv.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "905ff590-6fca-4cbd-b14f-1013ba462b33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.4288956429494437\n",
      "Testing Score: 0.31460017559065245\n"
     ]
    }
   ],
   "source": [
    "print(f'Training Score: {ridge_cv.score(X_train_sc, y_train)}')\n",
    "print(f'Testing Score: {ridge_cv.score(X_test_sc, y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4caf5b9-35cc-4cfa-9a38-9b48cba2979b",
   "metadata": {},
   "source": [
    "## Lasso\n",
    "\n",
    "After running a Ridge model, I ran a Lasso Model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "6788b567-d222-4997-bbf7-b1a78dd3f43e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.22549615455731165\n",
      "Testing Score: 0.18210534907696907\n"
     ]
    }
   ],
   "source": [
    "lasso = Lasso()\n",
    "\n",
    "lasso.fit(X_train_sc, y_train)\n",
    "\n",
    "print(f'Training Score: {lasso.score(X_train_sc, y_train)}')\n",
    "print(f'Testing Score: {lasso.score(X_test_sc, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d0d78b47-8936-4e27-8466-f088c278586c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LassoCV(alphas=array([0.001     , 0.00107227, 0.00114976, 0.00123285, 0.00132194,\n",
       "       0.00141747, 0.00151991, 0.00162975, 0.00174753, 0.00187382,\n",
       "       0.00200923, 0.00215443, 0.00231013, 0.00247708, 0.00265609,\n",
       "       0.00284804, 0.00305386, 0.00327455, 0.00351119, 0.00376494,\n",
       "       0.00403702, 0.00432876, 0.00464159, 0.00497702, 0.0053367 ,\n",
       "       0.00572237, 0.00613591, 0.00657933, 0.0070548 , 0.00756463,\n",
       "       0.008...\n",
       "       0.09326033, 0.1       , 0.10722672, 0.1149757 , 0.12328467,\n",
       "       0.13219411, 0.14174742, 0.15199111, 0.16297508, 0.17475284,\n",
       "       0.18738174, 0.2009233 , 0.21544347, 0.23101297, 0.24770764,\n",
       "       0.26560878, 0.28480359, 0.30538555, 0.32745492, 0.35111917,\n",
       "       0.37649358, 0.40370173, 0.43287613, 0.46415888, 0.49770236,\n",
       "       0.53366992, 0.57223677, 0.61359073, 0.65793322, 0.70548023,\n",
       "       0.75646333, 0.81113083, 0.869749  , 0.93260335, 1.        ]),\n",
       "        max_iter=10000)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l_alphas = np.logspace(-3, 0, 100)\n",
    "lasso_cv = LassoCV(alphas = l_alphas, max_iter = 10000)\n",
    "lasso_cv.fit(X_train_sc, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dcdf21f1-a469-4e4f-881e-f0fa4ac32fda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.046415888336127795"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_cv.alpha_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8ec90d7f-1bd0-476b-b604-576291a8e393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.4359546339755044\n",
      "Testing Score: 0.32456717498780796\n"
     ]
    }
   ],
   "source": [
    "print(f'Training Score: {lasso_cv.score(X_train_sc, y_train)}')\n",
    "print(f'Testing Score: {lasso_cv.score(X_test_sc, y_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "171c1afe-7b27-4812-a15e-150bf32e771f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.        ,  0.02325777,  0.        ,  0.        ,  0.        ,\n",
       "        0.09274581,  0.37761832,  0.00745193,  0.55719608,  0.        ,\n",
       "        0.        , -0.        ,  0.01709271,  0.05929859,  0.        ,\n",
       "        0.        , -0.        , -0.10450148,  0.        ,  0.11502586,\n",
       "        0.        , -0.        ,  0.        , -0.        ,  0.        ,\n",
       "        0.32949498,  0.16638357,  0.        ,  0.05667695, -0.        ,\n",
       "        0.        ,  0.21506551,  0.15632405, -0.        ,  0.        ,\n",
       "       -0.23158649])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_coefs = lasso_cv.coef_\n",
    "lasso_coefs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a3c6a146-e24b-4ea3-adec-3b51b09f9f93",
   "metadata": {},
   "outputs": [],
   "source": [
    "lasso_coefs = pd.Series(lasso_coefs, X.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7eedf90c-36b3-44db-85af-80a464e21d1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "YBC            0.557196\n",
       "TD             0.377618\n",
       "Yds_-1_year    0.329495\n",
       "Yds_-2_year    0.215066\n",
       "TD_-1_year     0.166384\n",
       "TD_-2_year     0.156324\n",
       "ProBowl        0.115026\n",
       "Yds            0.092746\n",
       "BrkTkl         0.059299\n",
       "GS_-2_year     0.056677\n",
       "G              0.023258\n",
       "ADOT           0.017093\n",
       "1D             0.007452\n",
       "Int           -0.104501\n",
       "YrsPlayed     -0.231586\n",
       "dtype: float64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lasso_coefs[lasso_coefs != 0].sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85bc786d-25a8-43a2-9aae-74672483dd76",
   "metadata": {},
   "source": [
    "One Standard Deviation increase in Yds will increase target yard projections by 141.745"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5626279-8fb6-413e-8c0b-95d90892ce8c",
   "metadata": {},
   "source": [
    "## ElasticNet\n",
    "\n",
    "Because I ran both a Ridge and a Lasso model, I wanted to also run an ElasticNet, which combines the previous two models to hopefully form a better model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "34a6b6a9-b78a-4e5e-bd58-2827d71b8604",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.4292771351798569\n",
      "Testing Score: 0.31580381690577597\n"
     ]
    }
   ],
   "source": [
    "enet_alphas = np.linspace(0.5, 1.0, 100)\n",
    "enet_ratio = 0.5\n",
    "enet_model = ElasticNetCV(alphas = enet_alphas, l1_ratio = enet_ratio, cv=5, max_iter = 100000)\n",
    "enet_model = enet_model.fit(X_train, y_train)\n",
    "\n",
    "print(f'Training Score: {enet_model.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {enet_model.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "365778c6-084d-43c9-a023-5db590139524",
   "metadata": {},
   "source": [
    "Afterwards, I tried adding Polynomial features to both the Ridge, Lasso, and ElasticNet models, but it either made them worse, or threw a bunch of conversion errors, so I decided against using them."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac595bb-8edb-4a7a-9f5e-aaff508f0b34",
   "metadata": {},
   "source": [
    "## Principal Component Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2f0f131a-f9bd-457d-af02-ecd8c422c584",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(columns = ['Tm', 'Pos', 'Player-additional', 'Tgt_target', 'Rec_target', 'Yds_target', 'TD_target', 'Year'])\n",
    "y = df['TD_target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "927177cd-0ad5-49e5-bd7b-41abec48a04f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c617d813-242e-4cc1-9539-c0cb19a26315",
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train_sc = ss.fit_transform(X_train)\n",
    "X_test_sc = ss.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "db72082e-2a9f-4480-9f50-fffcd123f5cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explained variance (first 20 components): [0.419 0.084 0.081 0.047 0.045 0.043 0.037 0.033 0.028 0.025 0.022 0.017\n",
      " 0.016 0.014 0.013 0.011 0.01  0.008 0.008 0.007]\n",
      "Cumulative explained variance (first 20 components): [0.419 0.503 0.584 0.631 0.676 0.719 0.756 0.788 0.816 0.841 0.863 0.88\n",
      " 0.896 0.911 0.924 0.935 0.945 0.953 0.961 0.968]\n",
      "Training Score: 0.4465\n",
      "Testing Score: 0.3236\n"
     ]
    }
   ],
   "source": [
    "pca = PCA(n_components = 28, random_state = 7)\n",
    "\n",
    "X_train_pca = pca.fit_transform(X_train_sc)\n",
    "X_test_pca = pca.transform(X_test_sc)\n",
    "\n",
    "lr2 = LinearRegression()\n",
    "\n",
    "lr2.fit(X_train_pca, y_train)\n",
    "\n",
    "var_exp = pca.explained_variance_ratio_\n",
    "print(f'Explained variance (first 20 components): {np.round(var_exp[:20], 3)}')\n",
    "cum_var_exp = np.cumsum(var_exp)\n",
    "print(f'Cumulative explained variance (first 20 components): {np.round(cum_var_exp[:20], 3)}')\n",
    "\n",
    "print(f'Training Score: {round(lr2.score(X_train_pca, y_train),4)}')\n",
    "print(f'Testing Score: {round(lr2.score(X_test_pca, y_test),4)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6291aef5-db2c-44c4-8736-1fb58221bb79",
   "metadata": {},
   "source": [
    "When running the PCA model, I was playing around with the number of components to use, and ended up finding my best result using the first 28 components."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18388f71-5679-404e-9e7b-41d3ff82853c",
   "metadata": {},
   "source": [
    "## Ensemble Model\n",
    "\n",
    "Now that we've tried a number of models, I will put some of our better scoring models together in a stacking model and see if we can get an even better score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d711acbf-b0dd-4396-ae4c-8e7b9997b765",
   "metadata": {},
   "outputs": [],
   "source": [
    "level1_models = [\n",
    "    ('rf', RandomForestRegressor()),\n",
    "    ('gb', GradientBoostingRegressor()),\n",
    "    ('ada', AdaBoostRegressor()),\n",
    "    ('lasso_pipe', Pipeline([\n",
    "        ('ss', StandardScaler()),\n",
    "        ('lasso', LassoCV())\n",
    "    ]))\n",
    "]\n",
    "\n",
    "stack = StackingRegressor(estimators = level1_models,\n",
    "                          final_estimator = LinearRegression())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e69e23b3-1c31-42ee-94f3-16def258b758",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3818501775419284"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(stack, X_train, y_train).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e145c3f8-722b-4003-bd83-3e371541229b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StackingRegressor(estimators=[('rf', RandomForestRegressor()),\n",
       "                              ('gb', GradientBoostingRegressor()),\n",
       "                              ('ada', AdaBoostRegressor()),\n",
       "                              ('lasso_pipe',\n",
       "                               Pipeline(steps=[('ss', StandardScaler()),\n",
       "                                               ('lasso', LassoCV())]))],\n",
       "                  final_estimator=LinearRegression())"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stack.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1dae1f6c-95c8-4b1a-8fd2-da516824c890",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score: 0.3323834324495176\n",
      "Testing Score: 0.3136380043607936\n"
     ]
    }
   ],
   "source": [
    "print(f'Training Score: {stack.score(X_train, y_train)}')\n",
    "print(f'Testing Score: {stack.score(X_test, y_test)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d17047b-e114-4b82-8610-8a9f6a25c35b",
   "metadata": {},
   "source": [
    "Similar to our receptions, it appears that our RandomForest model ended up being the highest scoring model, but the stacking model is a lot less overfit than the RandomForest model, so we'll use this for our Touchdown predictions model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5aa5cac0-4f64-46fc-a73c-c3eb2ede7029",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}