notebooks/3_advanced_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Advanced models and Hyperparameter Tuning\n",
    "\n",
    "We tested several different advanced models.\n",
    "\n",
    "The best performing model was a Gradient Boosted Tree model\n",
    "from the LightGBM library.\n",
    "\n",
    "We use the root mean square percentage error as a metric (RMSPE)\n",
    "\n",
    "![](../assets/rmspe.png)\n",
    "\n",
    "| Model              | RMSPE |\n",
    "|--------------------|-------|\n",
    "| Naive model (mean) | 62.03 |\n",
    "| Linear Regression  | 22.81 |\n",
    "| Random Forest      | 17.07 |\n",
    "| Light GBM          | 12.38 |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "matplotlib.rcParams['figure.dpi']=150\n",
    "\n",
    "# Load scripts from parent path\n",
    "import sys, os\n",
    "sys.path.insert(0, os.path.abspath('..'))\n",
    "\n",
    "# Ignore some future warnings triggered when training\n",
    "import warnings\n",
    "warnings.filterwarnings(action=\"ignore\", category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Store</th>\n",
       "      <th>DayOfWeek</th>\n",
       "      <th>Sales</th>\n",
       "      <th>Promo</th>\n",
       "      <th>StateHoliday</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>week</th>\n",
       "      <th>month</th>\n",
       "      <th>end_of_month</th>\n",
       "      <th>beginning_of_month</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>Assortment</th>\n",
       "      <th>CompetitionDistance</th>\n",
       "      <th>Store_Sales_mean</th>\n",
       "      <th>Store_Customers_mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>353.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3139.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.877078</td>\n",
       "      <td>b</td>\n",
       "      <td>b</td>\n",
       "      <td>900.0</td>\n",
       "      <td>4139.474576</td>\n",
       "      <td>1153.783333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>335.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.877078</td>\n",
       "      <td>b</td>\n",
       "      <td>a</td>\n",
       "      <td>90.0</td>\n",
       "      <td>12845.896552</td>\n",
       "      <td>2384.271186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>512.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2646.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.877078</td>\n",
       "      <td>b</td>\n",
       "      <td>b</td>\n",
       "      <td>590.0</td>\n",
       "      <td>3725.649123</td>\n",
       "      <td>888.627119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>494.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3113.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.877078</td>\n",
       "      <td>b</td>\n",
       "      <td>a</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>7079.150000</td>\n",
       "      <td>1010.583333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>530.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2907.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.877078</td>\n",
       "      <td>a</td>\n",
       "      <td>c</td>\n",
       "      <td>18160.0</td>\n",
       "      <td>2260.783333</td>\n",
       "      <td>333.610169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Store  DayOfWeek   Sales  Promo  StateHoliday  SchoolHoliday  week  month  \\\n",
       "0  353.0        2.0  3139.0    0.0           1.0            1.0     1      1   \n",
       "1  335.0        2.0  2401.0    0.0           1.0            1.0     1      1   \n",
       "2  512.0        2.0  2646.0    0.0           1.0            1.0     1      1   \n",
       "3  494.0        2.0  3113.0    0.0           1.0            1.0     1      1   \n",
       "4  530.0        2.0  2907.0    0.0           1.0            1.0     1      1   \n",
       "\n",
       "   end_of_month  beginning_of_month StoreType Assortment  CompetitionDistance  \\\n",
       "0      0.000001            0.877078         b          b                900.0   \n",
       "1      0.000001            0.877078         b          a                 90.0   \n",
       "2      0.000001            0.877078         b          b                590.0   \n",
       "3      0.000001            0.877078         b          a               1260.0   \n",
       "4      0.000001            0.877078         a          c              18160.0   \n",
       "\n",
       "   Store_Sales_mean  Store_Customers_mean  \n",
       "0       4139.474576           1153.783333  \n",
       "1      12845.896552           2384.271186  \n",
       "2       3725.649123            888.627119  \n",
       "3       7079.150000           1010.583333  \n",
       "4       2260.783333            333.610169  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scripts.processing as scr\n",
    "train_raw = scr.load_train_data()\n",
    "train = scr.add_week_month_info(train_raw)\n",
    "train = scr.add_beginning_end_month(train)\n",
    "train = scr.process_data(train)\n",
    "train = scr.add_store_info(train)\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare train/test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.copy(deep=True).drop(columns=[\"Sales\"])\n",
    "y = train.loc[:, \"Sales\"]\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boosted Tree Model\n",
    "\n",
    "We found that a Gradient Boosted Tree model had good performance on the dataset.\n",
    "\n",
    "Here we show the final model we found."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import category_encoders as ce\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "target_encode  = ce.TargetEncoder(cols = ['Store','StoreType','Assortment'])\n",
    "ordinal_encode = ce.OrdinalEncoder(cols=['StateHoliday'])\n",
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create model, train and evaluate\n",
    "\n",
    "We create a pipeline of the model and encoders."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11.478737148311913"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lightgbm import LGBMRegressor\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from scripts.processing import metric\n",
    "\n",
    "# Define the model\n",
    "reg = LGBMRegressor(\n",
    "        n_estimators=500,\n",
    "        max_depth=25,\n",
    "        num_leaves=80\n",
    "      )\n",
    "\n",
    "# Build the pipeline\n",
    "pipe_lgbm = Pipeline(steps=[\n",
    "                ('ordinal_encode', ordinal_encode), \n",
    "                ('target_encode', target_encode),\n",
    "                ('model', reg)]\n",
    "                )\n",
    "\n",
    "pipe_lgbm.fit(X_train, y_train)\n",
    "\n",
    "y_pred_train = pipe_lgbm.predict(X_train)\n",
    "y_pred = pipe_lgbm.predict(X_test)\n",
    "metric(y_test.values, y_pred)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameter Optimization\n",
    "\n",
    "We also performed hyperparameter optimziation to fine-tune the model.\n",
    "\n",
    "In order to select the best hyperparameters for our pipeline, we performed a grid search on a number of parameters. \n",
    "The hyperparameters corresponding the model that we explored were: n_estimators, max_depth, num_leaves and additionally the set of features to be target encoded."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example parameter combinations that were tested**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "   'model__n_estimators': [100, 500, 900, 5000],\n",
    "   'model__max_depth': [15, 30, 60, 100],\n",
    "   'model__num_leaves': [30, 60, 120, 300],\n",
    "   'target_encode__cols':\n",
    "   [\n",
    "      ['Store','StoreType','Assortment','StateHoliday','week','month','beginning_of_month','end_of_month'],\n",
    "      ['Store','StoreType','Assortment','StateHoliday','week','month'],\n",
    "      ['Store','StoreType','Assortment']\n",
    "   ]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Final best parameter combination**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 192 candidates, totalling 960 fits\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from lightgbm import LGBMRegressor\n",
    "model = LGBMRegressor()\n",
    "\n",
    "\n",
    "pipe = Pipeline(steps=[ \n",
    "                ('target_encode', target_encode),\n",
    "                ('scaler',scaler),\n",
    "                ('model',model)])\n",
    "\n",
    "\n",
    "regLGBMGridSearch = GridSearchCV(pipe, parameters, cv=5, n_jobs=-1, verbose=1, error_score='raise')\n",
    "grid_search = regLGBMGridSearch.fit(X_train, y_train) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter (CV score=0.944):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('target_encode',\n",
       "                 TargetEncoder(cols=['Store', 'StoreType', 'Assortment'])),\n",
       "                ('scaler', StandardScaler()),\n",
       "                ('model',\n",
       "                 LGBMRegressor(max_depth=15, n_estimators=5000,\n",
       "                               num_leaves=60))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Best parameter (CV score=%0.3f):\" % grid_search.best_score_)\n",
    "grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSPE metric on test set: 10.67\n"
     ]
    }
   ],
   "source": [
    "from scripts.processing import metric\n",
    "y_pred = grid_search.best_estimator_.predict(X_test)\n",
    "error_regLGBM = metric(y_test.values, y_pred)\n",
    "print(f\"RMSPE metric on test set: {error_regLGBM:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final Training\n",
    "\n",
    "We train the final model on the whole training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = grid_search.best_estimator_\n",
    "best_model_wholeDataRefitted = best_model.fit(X,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - Saving pipeline \"LGBM_hyperparam_optim_5\" at:\n",
      "../data/trained_pipelines/pipeline_LGBM_hyperparam_optim_5.p\n"
     ]
    }
   ],
   "source": [
    "from scripts.pipeline import save_pipeline\n",
    "pipe = save_pipeline(pipeline=best_model_wholeDataRefitted, name='LGBM_hyperparam_optim_5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "0ff12d4eed094120c91f1ffe1dee8a0b8612ad4a9cafaf463216d72bd5198085"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}