Own_Try9.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMIepsH4BLxh9ANAvE1/hWD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KenichiQaz/ML_Learning/blob/main/Own_Try9.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Chapter 9"
      ],
      "metadata": {
        "id": "gBYND5lFIWCt"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EuxVd97P8IZP",
        "outputId": "20f27234-a608-4b01-caea-64a00f9bd4fd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: dtreeviz in /usr/local/lib/python3.7/dist-packages (1.3.7)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (1.3.5)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (3.2.2)\n",
            "Requirement already satisfied: colour in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (0.1.5)\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (3.6.4)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (1.0.2)\n",
            "Requirement already satisfied: graphviz>=0.9 in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (0.10.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from dtreeviz) (1.21.6)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->dtreeviz) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->dtreeviz) (1.4.4)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->dtreeviz) (3.0.9)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->dtreeviz) (0.11.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->dtreeviz) (4.1.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->dtreeviz) (1.15.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->dtreeviz) (2022.2.1)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->dtreeviz) (0.7.1)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->dtreeviz) (1.4.1)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from pytest->dtreeviz) (22.1.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from pytest->dtreeviz) (57.4.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->dtreeviz) (8.14.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->dtreeviz) (1.11.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->dtreeviz) (3.1.0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->dtreeviz) (1.7.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->dtreeviz) (1.1.0)\n"
          ]
        }
      ],
      "source": [
        "# Tabular Modeling\n",
        "!pip install dtreeviz\n",
        "\n",
        "from pandas.api.types import is_string_dtype, is_numeric_dtype, is_categorical_dtype\n",
        "from fastai.tabular.all import *\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from sklearn.tree import DecisionTreeRegressor\n",
        "from dtreeviz.trees import *\n",
        "from IPython.display import Image, display_svg, SVG\n",
        "\n",
        "pd.options.display.max_rows = 20\n",
        "pd.options.display.max_columns = 8"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# kaggle login run once\n",
        "\n",
        "cred_path = Path('~/.kaggle/kaggle.json').expanduser()\n",
        "if not cred_path.exists():\n",
        "    cred_path.parent.mkdir(exist_ok=True)\n",
        "    cred_path.write_text(creds)\n",
        "    cred_path.chmod(0o600)"
      ],
      "metadata": {
        "id": "klOXyS6B9dKa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "ebdf1333-b812-4fd9-e64e-330d5a1955f1"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-4-755796ce2b6a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcred_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mcred_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmkdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexist_ok\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mcred_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcreds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mcred_path\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchmod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0o600\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'creds' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Pick a path to download the dataset to\n",
        "comp = 'bluebook-for-bulldozers'\n",
        "path = URLs.path(comp)\n",
        "path\n",
        "Path.BASE_PATH = path"
      ],
      "metadata": {
        "id": "qAWdMkth-GaY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Download dataset\n",
        "from kaggle import api\n",
        "\n",
        "if not path.exists():\n",
        "    path.mkdir(parents=true)\n",
        "    api.competition_download_cli(comp, path=path)\n",
        "    shutil.unpack_archive(str(path/f'{comp}.zip'), str(path))\n",
        "\n",
        "path.ls(file_type='text')"
      ],
      "metadata": {
        "id": "Y0yBBP3I-JsZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Show sample of data\n",
        "df = pd.read_csv(path/'TrainAndValid.csv', low_memory=False)\n",
        "df.columns"
      ],
      "metadata": {
        "id": "s-CXjUtD-tbA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Data Manipulation\n",
        "df['ProductSize'].unique()\n",
        "sizes = 'Large','Large / Medium','Medium','Small','Mini','Compact'\n",
        "df['ProductSize'] = df['ProductSize'].astype('category')\n",
        "df['ProductSize'].cat.set_categories(sizes, ordered=True, inplace=True)\n",
        "dep_var = 'SalePrice'\n",
        "df[dep_var] = np.log(df[dep_var])"
      ],
      "metadata": {
        "id": "sLECIP7uBswV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Potentially usefull categorical data\n",
        "df = add_datepart(df, 'saledate')\n",
        "df_test = pd.read_csv(path/'Test.csv', low_memory=False)\n",
        "df_test = add_datepart(df_test, 'saledate')\n",
        "' '.join(o for o in df.columns if o.startswith('sale'))"
      ],
      "metadata": {
        "id": "oo3EgV5LDfr1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Handle missing data or strings\n",
        "procs = [Categorify, FillMissing]"
      ],
      "metadata": {
        "id": "MUteW2OlDn5Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Split Training data\n",
        "cond = (df.saleYear<2011) | (df.saleMonth<10)\n",
        "train_idx = np.where( cond)[0]\n",
        "valid_idx = np.where(~cond)[0]\n",
        "\n",
        "splits = (list(train_idx),list(valid_idx))\n",
        "\n",
        "cont,cat = cont_cat_split(df, 1, dep_var=dep_var)\n",
        "to = TabularPandas(df, procs, cat, cont, y_names=dep_var, splits=splits)"
      ],
      "metadata": {
        "id": "pN-pTvYWD_9A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#show length of training and validation data\n",
        "len(to.train),len(to.valid)"
      ],
      "metadata": {
        "id": "YR5Nhc8_EVjs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Mapping to classes\n",
        "to.classes['ProductSize']"
      ],
      "metadata": {
        "id": "GOUHMd26EmWw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# saving a project\n",
        "save_pickle(path/'to.pkl',to)"
      ],
      "metadata": {
        "id": "Yk6CkmTETg_E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Loading a project\n",
        "to = load_pickle(path/'to.pkl')"
      ],
      "metadata": {
        "id": "rlVUvcWTTmIt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Defining Variables\n",
        "xs,y = to.train.xs,to.train.y\n",
        "valid_xs,valid_y = to.valid.xs,to.valid.y"
      ],
      "metadata": {
        "id": "glbAuVB_EstJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Decision tree\n",
        "m = DecisionTreeRegressor(max_leaf_nodes=4)\n",
        "m.fit(xs, y);"
      ],
      "metadata": {
        "id": "FM7qW-4hE0Dx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Show the tree\n",
        "draw_tree(m, xs, size=10, leaves_parallel=True, precision=2)"
      ],
      "metadata": {
        "id": "xyazPkNUE6rn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# show the same information using Terence Parr's dtreeviz library\n",
        "samp_idx = np.random.permutation(len(y))[:500]\n",
        "dtreeviz(m, xs.iloc[samp_idx], y.iloc[samp_idx], xs.columns, dep_var,\n",
        "        fontname='DejaVu Sans', scale=1.6, label_fontsize=10,\n",
        "        orientation='LR')"
      ],
      "metadata": {
        "id": "c855KE_sG_za"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# change starting visualization date to make the charts more useful\n",
        "xs.loc[xs['YearMade']<1900, 'YearMade'] = 1960\n",
        "valid_xs.loc[valid_xs['YearMade']<1900, 'YearMade'] = 1960"
      ],
      "metadata": {
        "id": "bPnAgbmfHLip"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Redraw charts\n",
        "m = DecisionTreeRegressor(max_leaf_nodes=4).fit(xs, y)\n",
        "\n",
        "dtreeviz(m, xs.iloc[samp_idx], y.iloc[samp_idx], xs.columns, dep_var,\n",
        "        fontname='DejaVu Sans', scale=1.6, label_fontsize=10,\n",
        "        orientation='LR')"
      ],
      "metadata": {
        "id": "YofEOtA7HWXB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Build a bigger tree\n",
        "m = DecisionTreeRegressor()\n",
        "m.fit(xs, y);\n"
      ],
      "metadata": {
        "id": "4cNnlgbWHmJH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check squared error of model (m_rmse)\n",
        "def r_mse(pred,y): return round(math.sqrt(((pred-y)**2).mean()), 6)\n",
        "def m_rmse(m, xs, y): return r_mse(m.predict(xs), y)\n",
        "m_rmse(m, xs, y)\n",
        "m_rmse(m, valid_xs, valid_y)"
      ],
      "metadata": {
        "id": "dv04X6UcHqvc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#fixing the overfitting\n",
        "m = DecisionTreeRegressor(min_samples_leaf=25)\n",
        "m.fit(to.train.xs, to.train.y)\n",
        "m_rmse(m, xs, y), m_rmse(m, valid_xs, valid_y)"
      ],
      "metadata": {
        "id": "jNTo-CfIIF6d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating a random forest\n",
        "def rf(xs, y, n_estimators=40, max_samples=200_000,\n",
        "       max_features=0.5, min_samples_leaf=5, **kwargs):\n",
        "    return RandomForestRegressor(n_jobs=-1, n_estimators=n_estimators,\n",
        "        max_samples=max_samples, max_features=max_features,\n",
        "        min_samples_leaf=min_samples_leaf, oob_score=True).fit(xs, y)\n",
        "m = rf(xs, y);"
      ],
      "metadata": {
        "id": "N7badiosIfNE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Predictors\n",
        "preds = np.stack([t.predict(valid_xs) for t in m.estimators_])\n",
        "r_mse(preds.mean(0), valid_y)\n",
        "plt.plot([r_mse(preds[:i+1].mean(0), valid_y) for i in range(40)]);"
      ],
      "metadata": {
        "id": "I1EQoNIqIz8M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# OOB prediction\n",
        "r_mse(m.oob_prediction_, y)"
      ],
      "metadata": {
        "id": "5awbNEDeJNuC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculating feature importance\n",
        "def rf_feat_importance(m, df):\n",
        "    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}\n",
        "                       ).sort_values('imp', ascending=False)\n",
        "fi = rf_feat_importance(m, xs)"
      ],
      "metadata": {
        "id": "z6VSJu4MJixU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Ploting Feature Importance\n",
        "def plot_fi(fi):\n",
        "    return fi.plot('cols', 'imp', 'barh', figsize=(12,7), legend=False)\n",
        "\n",
        "plot_fi(fi[:30]);"
      ],
      "metadata": {
        "id": "47ibGLXxJsG4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Removing low importance columns\n",
        "to_keep = fi[fi.imp>0.005].cols"
      ],
      "metadata": {
        "id": "_1-3i91-Jzrl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Retrain the model\n",
        "xs_imp = xs[to_keep]\n",
        "valid_xs_imp = valid_xs[to_keep]\n",
        "m = rf(xs_imp, y)"
      ],
      "metadata": {
        "id": "NjHTaPhMJ6Xb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Plot new feature importance chart\n",
        "plot_fi(rf_feat_importance(m, xs_imp));"
      ],
      "metadata": {
        "id": "NdnZ07VhKCe7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Showing redundencies\n",
        "cluster_columns(xs_imp)"
      ],
      "metadata": {
        "id": "cOZeFzPbKWCk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Remove redundencies\n",
        "def get_oob(df):\n",
        "    m = RandomForestRegressor(n_estimators=40, min_samples_leaf=15,\n",
        "        max_samples=50000, max_features=0.5, n_jobs=-1, oob_score=True)\n",
        "    m.fit(df, y)\n",
        "    return m.oob_score_\n",
        "\n",
        "# Redundent Variables\n",
        "{c:get_oob(xs_imp.drop(c, axis=1)) for c in (\n",
        "    'saleYear', 'saleElapsed', 'ProductGroupDesc','ProductGroup',\n",
        "    'fiModelDesc', 'fiBaseModel',\n",
        "    'Hydraulics_Flow','Grouser_Tracks', 'Coupler_System')}"
      ],
      "metadata": {
        "id": "edctEvuSKa7H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Drop Closely aligned pairs\n",
        "to_drop = ['saleYear', 'ProductGroupDesc', 'fiBaseModel', 'Grouser_Tracks']\n",
        "get_oob(xs_imp.drop(to_drop, axis=1))"
      ],
      "metadata": {
        "id": "4EigKaDWKusV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# create DataFrames\n",
        "xs_final = xs_imp.drop(to_drop, axis=1)\n",
        "valid_xs_final = valid_xs_imp.drop(to_drop, axis=1)\n",
        "\n",
        "# Save data\n",
        "save_pickle(path/'xs_final.pkl', xs_final)\n",
        "save_pickle(path/'valid_xs_final.pkl', valid_xs_final)\n",
        "\n",
        "# Load data\n",
        "xs_final = load_pickle(path/'xs_final.pkl')\n",
        "valid_xs_final = load_pickle(path/'valid_xs_final.pkl')"
      ],
      "metadata": {
        "id": "U9qCtiGhK6D9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Check RMSE\n",
        "m = rf(xs_final, y)\n",
        "m_rmse(m, xs_final, y), m_rmse(m, valid_xs_final, valid_y)"
      ],
      "metadata": {
        "id": "KHr1pQKaLvOf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Checking to see how common each category is\n",
        "p = valid_xs_final['ProductSize'].value_counts(sort=False).plot.barh()\n",
        "c = to.classes['ProductSize']\n",
        "plt.yticks(range(len(c)), c);"
      ],
      "metadata": {
        "id": "TGcNZWz2MAxR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Year made check\n",
        "ax = valid_xs_final['YearMade'].hist()"
      ],
      "metadata": {
        "id": "kaobV5UzMJzr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# partial dependence plot\n",
        "from sklearn.inspection import PartialDependenceDisplay\n",
        "\n",
        "fig,ax = plt.subplots(figsize=(12, 4))\n",
        "PartialDependenceDisplay.from_estimator(m, valid_xs_final, ['YearMade','ProductSize'],\n",
        "                        grid_resolution=20, ax=ax);"
      ],
      "metadata": {
        "id": "r9-bzmQJMTkT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Tree interpreter\n",
        "import warnings\n",
        "warnings.simplefilter('ignore', FutureWarning)\n",
        "\n",
        "from treeinterpreter import treeinterpreter\n",
        "from waterfall_chart import plot as waterfall"
      ],
      "metadata": {
        "id": "xNlQB1P2NAk7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Select a few rows from validation set and pass them to the interpreter\n",
        "row = valid_xs_final.iloc[:5]\n",
        "prediction,bias,contributions = treeinterpreter.predict(m, row.values)\n",
        "prediction[0], bias[0], contributions[0].sum()\n",
        "\n",
        "# Draw a waterfall chart of the independent variables\n",
        "waterfall(valid_xs_final.columns, contributions[0], threshold=0.08, \n",
        "          rotation_value=45,formatting='{:,.3f}');"
      ],
      "metadata": {
        "id": "H-BKuZKaNJ2Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Exploring Extrapolation Problem in Random Forests\n",
        "np.random.seed(42)\n",
        "x_lin = torch.linspace(0,20, steps=40)\n",
        "y_lin = x_lin + torch.randn_like(x_lin)\n",
        "plt.scatter(x_lin, y_lin);"
      ],
      "metadata": {
        "id": "F1KVNQDUNs8B"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert from vector to matrix\n",
        "xs_lin = x_lin.unsqueeze(1)\n",
        "x_lin.shape,xs_lin.shape\n",
        "x_lin[:,None].shape"
      ],
      "metadata": {
        "id": "2jeDU5oCN6uo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#create a random forest of the data and plot it\n",
        "m_lin = RandomForestRegressor().fit(xs_lin[:30],y_lin[:30])\n",
        "plt.scatter(x_lin, y_lin, 20)\n",
        "plt.scatter(x_lin, m_lin.predict(xs_lin), color='red', alpha=0.5);"
      ],
      "metadata": {
        "id": "u1xP-WhnOCeU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Find feature importance in total data set\n",
        "df_dom = pd.concat([xs_final, valid_xs_final])\n",
        "is_valid = np.array([0]*len(xs_final) + [1]*len(valid_xs_final))\n",
        "\n",
        "m = rf(df_dom, is_valid)\n",
        "\n",
        "m = rf(xs_final, y)\n",
        "print('orig', m_rmse(m, valid_xs_final, valid_y))\n",
        "\n",
        "for c in ('SalesID','saleElapsed','MachineID'):\n",
        "    m = rf(xs_final.drop(c,axis=1), y)\n",
        "    print(c, m_rmse(m, valid_xs_final.drop(c,axis=1), valid_y))"
      ],
      "metadata": {
        "id": "-24Uv3dGOppW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Removing SalesID and MachineID due to low importance\n",
        "time_vars = ['SalesID','MachineID']\n",
        "xs_final_time = xs_final.drop(time_vars, axis=1)\n",
        "valid_xs_time = valid_xs_final.drop(time_vars, axis=1)\n",
        "\n",
        "m = rf(xs_final_time, y)\n",
        "m_rmse(m, valid_xs_time, valid_y)"
      ],
      "metadata": {
        "id": "WKEcjZnnP-tO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Display salesyear graph\n",
        "xs['saleYear'].hist();"
      ],
      "metadata": {
        "id": "oS7l47zBQSMV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# result of training this dataset\n",
        "filt = xs['saleYear']>2004\n",
        "xs_filt = xs_final_time[filt]\n",
        "y_filt = y[filt]\n",
        "m = rf(xs_filt, y_filt)\n",
        "m_rmse(m, xs_filt, y_filt), m_rmse(m, valid_xs_time, valid_y)"
      ],
      "metadata": {
        "id": "kg9NiWTeQhC3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Neural network solution"
      ],
      "metadata": {
        "id": "WJGOz4OJSMg2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Building a NN to solve the problem\n",
        "df_nn = pd.read_csv(path/'TrainAndValid.csv', low_memory=False)\n",
        "df_nn['ProductSize'] = df_nn['ProductSize'].astype('category')\n",
        "df_nn['ProductSize'].cat.set_categories(sizes, ordered=True, inplace=True)\n",
        "df_nn[dep_var] = np.log(df_nn[dep_var])\n",
        "df_nn = add_datepart(df_nn, 'saledate')\n",
        "df_nn_final = df_nn[list(xs_final_time.columns) + [dep_var]]\n",
        "cont_nn,cat_nn = cont_cat_split(df_nn_final, max_card=9000, dep_var=dep_var)\n",
        "cont_nn"
      ],
      "metadata": {
        "id": "NNOEzVVuQrfj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Display the cardinality of each of the categorical variables\n",
        "df_nn_final[cat_nn].nunique()"
      ],
      "metadata": {
        "id": "uJkyPqfnQyKL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Removing unnessasary fiModelDescriptor category\n",
        "xs_filt2 = xs_filt.drop('fiModelDescriptor', axis=1)\n",
        "valid_xs_time2 = valid_xs_time.drop('fiModelDescriptor', axis=1)\n",
        "m2 = rf(xs_filt2, y_filt)\n",
        "m_rmse(m2, xs_filt2, y_filt), m_rmse(m2, valid_xs_time2, valid_y)\n",
        "cat_nn.remove('fiModelDescriptor')"
      ],
      "metadata": {
        "id": "gfajCw4DRKdQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# adding the normalize processor and setting the batch size and y range\n",
        "procs_nn = [Categorify, FillMissing, Normalize]\n",
        "to_nn = TabularPandas(df_nn_final, procs_nn, cat_nn, cont_nn,\n",
        "                      splits=splits, y_names=dep_var)\n",
        "\n",
        "dls = to_nn.dataloaders(1024)\n",
        "\n",
        "y = to_nn.train.y\n",
        "y.min(),y.max()"
      ],
      "metadata": {
        "id": "opZ7nkr2Rd55"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating learner\n",
        "learn = tabular_learner(dls, y_range=(8,12), layers=[500,250],\n",
        "                        n_out=1, loss_func=F.mse_loss)\n",
        "\n",
        "# Draw learner graph\n",
        "learn.lr_find()"
      ],
      "metadata": {
        "id": "52v_eRt1RyRc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Train the model for a few epochs\n",
        "learn.fit_one_cycle(5, 1e-2)"
      ],
      "metadata": {
        "id": "Pd7TIYV-SEOq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Save the model\n",
        "learn.save('nn')"
      ],
      "metadata": {
        "id": "HeCacQQ7SiFq"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}