fork-of-submit-i-am-just-kidding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "import seaborn as sns\n",
    "import datetime\n",
    "import tqdm\n",
    "import re\n",
    "import time\n",
    "from string import punctuation\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch, torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "sns.set_style('darkgrid')\n",
    "mpl.rcParams['figure.figsize'] = [15,10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kaggle.competitions import nflrush"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from string import punctuation\n",
    "import datetime\n",
    "import re\n",
    "import torch\n",
    "import math\n",
    "import tqdm\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(\n",
    "                    np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(\n",
    "                    np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(\n",
    "                    np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(\n",
    "                    np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(\n",
    "                    np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(\n",
    "                    np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose:\n",
    "        print(f'Mem. usage decreased to {end_mem} Mb ({100 * (start_mem - end_mem) / start_mem}% reduction)')\n",
    "    return df\n",
    "\n",
    "\n",
    "def crps(y_true, y_pred):\n",
    "    return np.mean(np.square(y_true - y_pred), axis=1)\n",
    "\n",
    "\n",
    "def yard_to_cdf(yard):\n",
    "    yard = np.round(yard).astype(int)\n",
    "    indices = yard + 99\n",
    "    cdfs = np.zeros((yard.shape[0], 199))\n",
    "    for i in range(len(cdfs)):\n",
    "        cdfs[i, indices[i]:] = 1\n",
    "    return cdfs\n",
    "\n",
    "\n",
    "def cdf_to_yard(cdf):\n",
    "    yard_index = (cdf == 1).argmax(axis=1)\n",
    "    yard = yard_index - 99\n",
    "    return yard\n",
    "\n",
    "\n",
    "def cdf_to_yard_torch(cdf):\n",
    "    yard_index = torch.sum((torch.as_tensor(cdf) <= 0), dim=1)\n",
    "    yard = yard_index - 99\n",
    "    return yard\n",
    "\n",
    "\n",
    "def crps_torch(y_true, y_pred):\n",
    "    y_true = torch.as_tensor(y_true)\n",
    "    y_pred = torch.as_tensor(y_pred)\n",
    "    return torch.mean((y_true - y_pred).pow(2), dim=1)\n",
    "\n",
    "\n",
    "def crps_loss(y_true, y_pred_pdf):\n",
    "    y_pred_cdf = torch.cumsum(torch.as_tensor(y_pred_pdf), dim=1)\n",
    "    return crps_torch(y_true, y_pred_cdf).mean()\n",
    "\n",
    "\n",
    "def crps_loss_cdf(y_true, y_pred_cdf):\n",
    "    return crps_torch(y_true, y_pred_cdf).mean()\n",
    "\n",
    "\n",
    "def clean_StadiumType(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    txt = txt.lower()\n",
    "    txt = ''.join([c for c in txt if c not in punctuation])\n",
    "    txt = re.sub(' +', ' ', txt)\n",
    "    txt = txt.strip()\n",
    "    txt = txt.replace('outside', 'outdoor')\n",
    "    txt = txt.replace('outdor', 'outdoor')\n",
    "    txt = txt.replace('outddors', 'outdoor')\n",
    "    txt = txt.replace('outdoors', 'outdoor')\n",
    "    txt = txt.replace('oudoor', 'outdoor')\n",
    "    txt = txt.replace('indoors', 'indoor')\n",
    "    txt = txt.replace('ourdoor', 'outdoor')\n",
    "    txt = txt.replace('retractable', 'rtr.')\n",
    "    return txt\n",
    "\n",
    "\n",
    "def transform_StadiumType(txt):\n",
    "    if pd.isna(txt):\n",
    "        return np.nan\n",
    "    if 'outdoor' in txt or 'open' in txt:\n",
    "        return 1\n",
    "    if 'indoor' in txt or 'closed' in txt:\n",
    "        return 0\n",
    "    return np.nan\n",
    "\n",
    "\n",
    "def str_to_seconds(txt):\n",
    "    txt = txt.split(':')\n",
    "    ans = int(txt[0]) * 60 + int(txt[1]) + int(txt[2]) / 60\n",
    "    return ans\n",
    "\n",
    "\n",
    "def str_to_float(txt):\n",
    "    try:\n",
    "        return float(txt)\n",
    "    except Exception as e:\n",
    "        return np.NaN\n",
    "\n",
    "\n",
    "def map_weather(txt):\n",
    "    ans = 1\n",
    "    if pd.isna(txt):\n",
    "        return 0\n",
    "    if 'partly' in txt:\n",
    "        ans *= 0.5\n",
    "    if 'climate controlled' in txt or 'indoor' in txt:\n",
    "        return ans * 3\n",
    "    if 'sunny' in txt or 'sun' in txt:\n",
    "        return ans * 2\n",
    "    if 'clear' in txt:\n",
    "        return ans\n",
    "    if 'cloudy' in txt:\n",
    "        return -ans\n",
    "    if 'rain' in txt or 'rainy' in txt:\n",
    "        return -2 * ans\n",
    "    if 'snow' in txt:\n",
    "        return -3 * ans\n",
    "    return 0\n",
    "\n",
    "\n",
    "def standartize_orientations(df):\n",
    "    \"\"\"\n",
    "        https://www.kaggle.com/cpmpml/initial-wrangling-voronoi-areas-in-python\n",
    "\n",
    "        Make sure the offensive team is always moving left to right.\n",
    "    \"\"\"\n",
    "    df['ToLeft'] = df.PlayDirection == \"left\"\n",
    "    df['TeamOnOffense'] = \"home\"\n",
    "    df.loc[df.PossessionTeam != df.HomeTeamAbbr, 'TeamOnOffense'] = \"away\"\n",
    "    df['IsOnOffense'] = df.Team == df.TeamOnOffense  # Is player on offense?\n",
    "    df['HomeOnOffense'] = (df['TeamOnOffense'] == 'home').astype(int)\n",
    "\n",
    "    df['YardLine_std'] = 100 - df.YardLine\n",
    "    df.loc[df.FieldPosition.fillna('') == df.PossessionTeam,\n",
    "           'YardLine_std'\n",
    "    ] = df.loc[df.FieldPosition.fillna('') == df.PossessionTeam,\n",
    "               'YardLine']\n",
    "    df['X_std'] = df.X\n",
    "    df.loc[df.ToLeft, 'X_std'] = 120 - df.loc[df.ToLeft, 'X']\n",
    "    df['Y_std'] = df.Y\n",
    "    df.loc[df.ToLeft, 'Y_std'] = 160 / 3 - df.loc[df.ToLeft, 'Y']\n",
    "\n",
    "    df['Dir_rad'] = np.mod(90 - df.Dir, 360) * math.pi / 180.0\n",
    "    df['Dir_std'] = df.Dir_rad\n",
    "    df.loc[df.ToLeft, 'Dir_std'] = np.mod(np.pi + df.loc[df.ToLeft, 'Dir_rad'],\n",
    "                                          2 * np.pi)\n",
    "\n",
    "    df['Orientation_rad'] = np.mod(df.Orientation, 360) * math.pi / 180.0\n",
    "\n",
    "    df.loc[df.Season >= 2018, 'Orientation_rad'\n",
    "    ] = np.mod(df.loc[df.Season >= 2018, 'Orientation'] - 90,\n",
    "               360) * math.pi / 180.0\n",
    "\n",
    "    df['Orientation_rad'] = np.mod(df.Orientation, 360) * math.pi / 180.0\n",
    "    df.loc[df.Season >= 2018, 'Orientation_rad'\n",
    "    ] = np.mod(df.loc[df.Season >= 2018, 'Orientation'] - 90,\n",
    "               360) * math.pi / 180.0\n",
    "    df['Orientation_std'] = df.Orientation_rad\n",
    "    df.loc[df.ToLeft, 'Orientation_std'] = np.mod(\n",
    "        math.pi + df.loc[df.ToLeft, 'Orientation_rad'], 2 * math.pi)\n",
    "\n",
    "    replace_cols = ['YardLine', 'X', 'Y', 'Dir', 'Orientation']\n",
    "    for col in replace_cols:\n",
    "        df[col] = df[col + '_std']\n",
    "        df.drop([col + '_std'], axis=1, inplace=True)\n",
    "\n",
    "    drop_cols = ['Dir_rad', 'Orientation_rad']\n",
    "    for col in drop_cols:\n",
    "        df.drop([col], axis=1, inplace=True)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def map_turf(df):\n",
    "    Turf = {'Field Turf': 'Artificial', 'A-Turf Titan': 'Artificial',\n",
    "            'Grass': 'Natural', 'UBU Sports Speed S5-M': 'Artificial',\n",
    "            'Artificial': 'Artificial', 'DD GrassMaster': 'Artificial',\n",
    "            'Natural Grass': 'Natural',\n",
    "            'UBU Speed Series-S5-M': 'Artificial', 'FieldTurf': 'Artificial',\n",
    "            'FieldTurf 360': 'Artificial', 'Natural grass': 'Natural',\n",
    "            'grass': 'Natural',\n",
    "            'Natural': 'Natural', 'Artifical': 'Artificial',\n",
    "            'FieldTurf360': 'Artificial', 'Naturall Grass': 'Natural',\n",
    "            'Field turf': 'Artificial',\n",
    "            'SISGrass': 'Artificial', 'Twenty-Four/Seven Turf': 'Artificial',\n",
    "            'natural grass': 'Natural'}\n",
    "    \n",
    "    turf_type = df['Turf'].map(Turf)\n",
    "    df['TurfIsNatural'] = (turf_type == 'Natural')\n",
    "    df = df.drop(['Turf'], axis=1)\n",
    "    return df\n",
    "\n",
    "def clean_abbrs(df):\n",
    "    # CAREFUL. What if a new team appears?\n",
    "    map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}\n",
    "    for abb in df['PossessionTeam'].unique():\n",
    "        map_abbr[abb] = abb\n",
    "\n",
    "    def safe_map(val):\n",
    "        if map_abbr.get(val):\n",
    "            return map_abbr[val]\n",
    "        else:\n",
    "            return val\n",
    "\n",
    "    df['PossessionTeam'] = df['PossessionTeam'].apply(safe_map)\n",
    "    df['HomeTeamAbbr'] = df['HomeTeamAbbr'].apply(safe_map)\n",
    "    df['VisitorTeamAbbr'] = df['VisitorTeamAbbr'].apply(safe_map)\n",
    "    df['FieldPosition'] = df['FieldPosition'].apply(safe_map)\n",
    "    return df\n",
    "\n",
    "def clean_weather(df):\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(\n",
    "        lambda x: x.lower().replace('mph', '').strip() if not pd.isna(x) else x)\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(lambda x: (int(x.split('-')[0]) + int(\n",
    "        x.split('-')[1])) / 2 if not pd.isna(x) and '-' in x else x)\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(\n",
    "        lambda x: (int(x.split()[0]) + int(x.split()[-1])) / 2 if not pd.isna(\n",
    "            x) and type(x) != float and 'gusts up to' in x else x)\n",
    "    df['WindSpeed'] = df['WindSpeed'].apply(str_to_float)\n",
    "\n",
    "    df = df.drop(['WindDirection'], axis=1)\n",
    "\n",
    "    df['GameWeather'] = df['GameWeather'].str.lower()\n",
    "    indoor = \"indoor\"\n",
    "    df['GameWeather'] = df['GameWeather'].apply(\n",
    "        lambda x: indoor if not pd.isna(x) and indoor in x else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(\n",
    "        lambda x: x.replace('coudy', 'cloudy').replace('clouidy',\n",
    "                                                       'cloudy').replace('party',\n",
    "                                                                         'partly') if not pd.isna(\n",
    "            x) else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(\n",
    "        lambda x: x.replace('clear and sunny', 'sunny and clear') if not pd.isna(\n",
    "            x) else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(\n",
    "        lambda x: x.replace('skies', '').replace(\"mostly\",\n",
    "                                                 \"\").strip() if not pd.isna(\n",
    "            x) else x)\n",
    "    df['GameWeather'] = df['GameWeather'].apply(map_weather)\n",
    "    return df\n",
    "\n",
    "# Encoding\n",
    "\n",
    "def encode_formations(df):\n",
    "    # Formation columns\n",
    "    df = pd.concat([df.drop(['OffenseFormation'], axis=1),\n",
    "                    pd.get_dummies(df['OffenseFormation'],\n",
    "                                   prefix='OffenseFormation')], axis=1)\n",
    "    # Filling missing dummy columns at test stage\n",
    "    expected_columns = ['OffenseFormation_ACE',\n",
    "                        'OffenseFormation_EMPTY',\n",
    "                        'OffenseFormation_JUMBO',\n",
    "                        'OffenseFormation_PISTOL',\n",
    "                        'OffenseFormation_SHOTGUN',\n",
    "                        'OffenseFormation_SINGLEBACK',\n",
    "                        'OffenseFormation_WILDCAT',\n",
    "                        'OffenseFormation_I_FORM']\n",
    "    for col in expected_columns:\n",
    "        if not col in df.columns:\n",
    "            df[col] = 0\n",
    "    return df\n",
    "\n",
    "def encode_personell(df):\n",
    "    # DefensePersonnel\n",
    "    counts = []\n",
    "    for i, val in df['DefensePersonnel'].str.split(',').iteritems():\n",
    "        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0,\n",
    "               'QB': 0}\n",
    "        if val is np.NaN:\n",
    "            counts.append({})\n",
    "            continue\n",
    "        for item in val:\n",
    "            name, number = item.strip().split(' ')[::-1]\n",
    "            row[name] = int(number)\n",
    "        counts.append(row)\n",
    "    defense_presonell_df = pd.DataFrame(counts)\n",
    "    defense_presonell_df.columns = ['defense_' + x for x in\n",
    "                                    defense_presonell_df.columns]\n",
    "    defense_presonell_df = defense_presonell_df.fillna(0).astype(int)\n",
    "    defense_presonell_df.index = df.index\n",
    "    df = pd.concat([df.drop(['DefensePersonnel'], axis=1), defense_presonell_df],\n",
    "                   axis=1)\n",
    "\n",
    "    # OffensePersonnel\n",
    "    counts = []\n",
    "    for i, val in df['OffensePersonnel'].str.split(',').iteritems():\n",
    "        row = {'OL': 0, 'RB': 0, 'TE': 0, 'WR': 0, 'DL': 0, 'DB': 0, 'LB': 0,\n",
    "               'QB': 0}\n",
    "        if val is np.NaN:\n",
    "            counts.append({})\n",
    "            continue\n",
    "        for item in val:\n",
    "            name, number = item.strip().split(' ')[::-1]\n",
    "            row[name] = int(number)\n",
    "        counts.append(row)\n",
    "    offense_personnel_df = pd.DataFrame(counts)\n",
    "    offense_personnel_df.columns = ['offense_' + x for x in\n",
    "                                    offense_personnel_df.columns]\n",
    "    offense_personnel_df = offense_personnel_df.fillna(0).astype(int)\n",
    "    offense_personnel_df.index = df.index\n",
    "    df = pd.concat([df.drop(['OffensePersonnel'], axis=1), offense_personnel_df],\n",
    "                   axis=1)\n",
    "    return df\n",
    "\n",
    "def engineer_features(df):\n",
    "    df['DefendersInTheBox_vs_Distance'] = (\n",
    "        df['DefendersInTheBox'] / df['Distance'])\n",
    "\n",
    "    df = add_phys_features(df)\n",
    "    return df\n",
    "\n",
    "def preprocess_features(df, verbose=False):\n",
    "    \"\"\"Accepts df like train data, returns cleaned, standartized and enriched df\"\"\"\n",
    "\n",
    "    df = clean_abbrs(df)\n",
    "\n",
    "    df = standartize_orientations(df)\n",
    "\n",
    "    df = map_turf(df)\n",
    "\n",
    "    df['StadiumType'] = df['StadiumType'].apply(clean_StadiumType)\n",
    "    df['StadiumTypeShort'] = df['StadiumType'].apply(transform_StadiumType)\n",
    "    df = df.drop(['StadiumType'], axis=1)\n",
    "\n",
    "    # df['HomeField'] = df['FieldPosition'] == df['HomeTeamAbbr']\n",
    "    # df['Field_eq_Possession'] = df['FieldPosition'] == df['PossessionTeam']\n",
    "    # df['HomePossesion'] = (df['PossessionTeam'] == df['HomeTeamAbbr'])\n",
    "\n",
    "    df['GameClock'] = df['GameClock'].apply(str_to_seconds)\n",
    "\n",
    "    df['PlayerHeight'] = df['PlayerHeight'].apply(\n",
    "        lambda x: 12 * int(x.split('-')[0]) + int(x.split('-')[1]))\n",
    "    df['PlayerBMI'] = 703 * (df['PlayerWeight'] / (df['PlayerHeight']) ** 2)\n",
    "\n",
    "    df['TimeHandoff'] = pd.to_datetime(df['TimeHandoff'], utc=True)\n",
    "    df['TimeSnap'] = pd.to_datetime(df['TimeSnap'], utc=True)\n",
    "    df['TimeDelta'] = (df['TimeHandoff'] - df['TimeSnap']).apply(\n",
    "        lambda x: x.total_seconds())\n",
    "    df['PlayerBirthDate'] = df['PlayerBirthDate'].apply(\n",
    "        lambda x: datetime.datetime.strptime(x, \"%m/%d/%Y\"))\n",
    "    df['PlayerBirthDate'] = pd.to_datetime(df['PlayerBirthDate'], utc=True)\n",
    "\n",
    "    seconds_in_year = 60 * 60 * 24 * 365.25\n",
    "    df['PlayerAge'] = (df['TimeHandoff'] - df['PlayerBirthDate']).apply(\n",
    "        lambda x: x.total_seconds()) / seconds_in_year\n",
    "\n",
    "    df = clean_weather(df)\n",
    "\n",
    "    df['IsRusher'] = df['NflId'] == df['NflIdRusher']\n",
    "    df['IsQB'] = df['Position'] == 'QB'\n",
    "\n",
    "    df = sort_df(df)\n",
    "\n",
    "    if df.shape[0] > 1000:\n",
    "        df = reduce_mem_usage(df, verbose=verbose)\n",
    "\n",
    "    return df\n",
    "\n",
    "def sort_df(df):\n",
    "    df.sort_values(by=['PlayId', 'IsOnOffense', 'IsRusher', 'IsQB', 'Position'], inplace=True)\n",
    "    return df\n",
    "\n",
    "def make_y(df):\n",
    "    y = np.zeros(shape=(df['Yards'][::22].shape[0], 199))\n",
    "    for i, yard in enumerate(df['Yards'][::22]):\n",
    "        y[i, yard + 99:] = np.ones(shape=(1, 100 - yard))\n",
    "    return y\n",
    "\n",
    "def compute_play_phys_features(play_id, play_df):\n",
    "    assert play_df.shape[0] == 22\n",
    "    play_index = play_df.index\n",
    "    offense_index = play_df.IsOnOffense\n",
    "    defense_index = ~play_df.IsOnOffense\n",
    "    offense_df = play_df[offense_index]\n",
    "    defense_df = play_df[defense_index]\n",
    "\n",
    "    offense_x_std = offense_df.X.std()\n",
    "    offense_y_std = offense_df.Y.std()\n",
    "    offense_mean_force = offense_df.Force.mean()\n",
    "\n",
    "    defense_x_std = defense_df.X.std()\n",
    "    defense_y_std = defense_df.Y.std()\n",
    "    defense_mean_force = defense_df.Force.mean()\n",
    "\n",
    "    rusher_row = offense_df[offense_df.IsRusher].iloc[0]\n",
    "\n",
    "    qb_row = offense_df[offense_df.Position == 'QB']\n",
    "    if not qb_row.empty:\n",
    "        qb_row = qb_row.iloc[0]\n",
    "\n",
    "    rusher_pos = rusher_row[['X', 'Y']].values[0]\n",
    "    qb_pos = qb_row[['X', 'Y']].values[0] if not qb_row.empty else None\n",
    "\n",
    "    rusher_dist_to_qb = np.linalg.norm(\n",
    "        rusher_pos - qb_pos) if not qb_row.empty else None\n",
    "\n",
    "    # Defender to rusher distances\n",
    "    defense_distances_to_runner = []\n",
    "    for row in defense_df.itertuples():\n",
    "        pos = np.array([row.X, row.Y])\n",
    "        defense_distances_to_runner.append(np.linalg.norm(rusher_pos - pos))\n",
    "    dist_to_rusher = defense_distances_to_runner = np.array(\n",
    "        defense_distances_to_runner)\n",
    "\n",
    "    time_to_rusher = defense_df['S'] / dist_to_rusher\n",
    "\n",
    "    defender_time_to_runner_min = time_to_rusher.min()\n",
    "\n",
    "    # closest defenders\n",
    "    closest_bytime_defender = \\\n",
    "        defense_df[time_to_rusher == defender_time_to_runner_min].iloc[0]\n",
    "\n",
    "    closest_bytime_defender_force_div_rusher_force = (\n",
    "        closest_bytime_defender.Force / rusher_row.Force)\n",
    "\n",
    "    closest_bytime_defender_speed_div_rusher_speed = closest_bytime_defender.S / rusher_row.S\n",
    "    closest_bytime_defender_acceleration_div_rusher_acceleration = closest_bytime_defender.A / rusher_row.A\n",
    "\n",
    "    # Add play features\n",
    "    new_play_features = {\n",
    "        'phys_offense_x_std': offense_x_std,\n",
    "        'phys_offense_y_std': offense_y_std,\n",
    "        'phys_offense_mean_force': offense_mean_force,\n",
    "        'phys_defense_x_std': defense_x_std,\n",
    "        'phys_defense_y_std': defense_y_std,\n",
    "        'phys_defense_mean_force': defense_mean_force,\n",
    "\n",
    "        'phys_rusher_dist_to_qb': rusher_dist_to_qb,\n",
    "        'phys_defender_time_to_runner_min': defender_time_to_runner_min,\n",
    "\n",
    "        'phys_closest_bytime_defender_force_div_rusher_force': closest_bytime_defender_force_div_rusher_force,\n",
    "\n",
    "        'phys_closest_bytime_defender_speed_div_rusher_speed': closest_bytime_defender_speed_div_rusher_speed,\n",
    "        'phys_closest_bytime_defender_acceleration_div_rusher_acceleration': closest_bytime_defender_acceleration_div_rusher_acceleration,\n",
    "    }\n",
    "    return new_play_features\n",
    "\n",
    "def compute_phys_features(df):\n",
    "    playid_index = df.PlayId.unique()\n",
    "    features_rows= []\n",
    "    for play_id in tqdm.tqdm(playid_index):\n",
    "        play_df = df[df.PlayId == play_id]\n",
    "        features_rows.append(compute_play_phys_features(play_id, play_df))\n",
    "\n",
    "    features_df = pd.DataFrame(features_rows, index=playid_index)\n",
    "    return features_df\n",
    "\n",
    "def compute_rusher_features(df, counters=None):\n",
    "    if counters: \n",
    "        (games_counter, plays_counter, last_game, plays_cur_game_counter) = counters\n",
    "    else:\n",
    "        games_counter = Counter()\n",
    "        plays_counter = Counter()\n",
    "        # yards_counter = Counter()\n",
    "\n",
    "        last_game = {}\n",
    "        plays_cur_game_counter = Counter()\n",
    "\n",
    "    feature_rows = []\n",
    "    for row in df[df.IsRusher].itertuples():\n",
    "        game_id = row.GameId\n",
    "        # yards = row.Yards\n",
    "        rusher_id = row.NflId\n",
    "\n",
    "        rusher_games = games_counter[rusher_id]\n",
    "        rusher_plays = plays_counter[rusher_id]\n",
    "        rusher_plays_current_game = plays_cur_game_counter[rusher_id] if game_id == last_game.get(rusher_id) else 0 # tiredness\n",
    "        # rusher_yards = yards_counter[rusher_id]\n",
    "\n",
    "        row = (rusher_id, rusher_plays, rusher_games, rusher_plays_current_game)\n",
    "\n",
    "        feature_rows.append(row)\n",
    "\n",
    "        if not last_game.get(rusher_id) or game_id != last_game[rusher_id]:\n",
    "            last_game[rusher_id] = game_id\n",
    "            games_counter[rusher_id] +=1\n",
    "            plays_cur_game_counter[rusher_id] = 0\n",
    "\n",
    "        plays_counter[rusher_id] += 1\n",
    "        plays_cur_game_counter[rusher_id] += 1\n",
    "        # yards_counter[rusher_id] += yards\n",
    "        \n",
    "    rusher_stats_features = pd.DataFrame(feature_rows, \n",
    "                                        index=df[df.IsRusher].PlayId,\n",
    "                                        columns = ['rusher_id', 'rusher_plays', 'rusher_games', 'rusher_plays_current_game'])\n",
    "    # rusher_stats_features['rusher_yards_per_game'] = (rusher_stats_features['rusher_yards'] / rusher_stats_features['rusher_games']).fillna(0)\n",
    "    # rusher_stats_features['rusher_yards_per_play'] = (rusher_stats_features['rusher_yards'] / rusher_stats_features['rusher_plays']).fillna(0)\n",
    "    rusher_stats_features['rusher_plays_per_game'] = (rusher_stats_features['rusher_plays'] / rusher_stats_features['rusher_games']).fillna(0)\n",
    "    \n",
    "    return rusher_stats_features, (games_counter, plays_counter, last_game, plays_cur_game_counter)\n",
    "\n",
    "def make_x(df, stats_counters=None, encode=True, fillna=True, verbose=True, fe_phys=True):\n",
    "    \"\"\"Input: source data, preprocesed, feature dfs.\n",
    "\n",
    "       Output: X dataframe\n",
    "    \"\"\"\n",
    "    source_play_id = df['PlayId']\n",
    "    df = df.copy()\n",
    "    df.index = source_play_id\n",
    "\n",
    "    # Encoding\n",
    "    if encode:\n",
    "        df = encode_formations(df)\n",
    "        df = encode_personell(df)\n",
    "\n",
    "    # Feature engineering\n",
    "\n",
    "    df['DefendersInTheBox_vs_Distance'] = (\n",
    "        df['DefendersInTheBox'] / df['Distance'])\n",
    "\n",
    "    df['Force'] = df['A'] * df['PlayerWeight']\n",
    "\n",
    "    \n",
    "    if fe_phys:\n",
    "        if verbose:\n",
    "            print('Computing phys features')\n",
    "        phys_features = compute_phys_features(df)\n",
    "\n",
    "    rusher_stats_features, rusher_stats_counters = compute_rusher_features(df, counters=stats_counters)\n",
    "    # Assemble X\n",
    "\n",
    "    # Drop unnececary rows, keep only rusher\n",
    "\n",
    "    df = df[df.IsRusher].copy()\n",
    "\n",
    "    # Drop unnececary columns\n",
    "\n",
    "    cols_to_drop = [\n",
    "        'Yards', 'PlayDirection', 'TeamOnOffense', 'NflId', 'NflIdRusher',\n",
    "        'TimeHandoff', 'TimeSnap', 'PlayerBirthDate', 'FieldPosition',\n",
    "        'DisplayName', 'PossessionTeam', 'PlayerCollegeName', 'Position',\n",
    "        'HomeTeamAbbr', 'VisitorTeamAbbr', 'Stadium', 'Location', 'GameId',\n",
    "        'PlayId', 'Team', 'IsRusher', 'IsQB', 'IsOnOffense', 'Toleft',\n",
    "        'HomeOnOffense', 'Temperature', 'Humidity', 'WindSpeed', 'GameWeather',\n",
    "        'WindDirection', 'ToLeft', 'X', 'Y', 'Dis', \n",
    "        'Orientation'\n",
    "    ]\n",
    "    cols_to_drop = list(set(cols_to_drop).intersection(set(list(df.columns))))\n",
    "    df.drop(cols_to_drop, axis=1, inplace=True)\n",
    "\n",
    "    if verbose:\n",
    "        print('Dropped cols:', cols_to_drop)\n",
    "\n",
    "    # Assemble players features\n",
    "    cols_player = ['S',\n",
    "                   'A',\n",
    "                   'Dir',\n",
    "                   'PlayerBMI',\n",
    "                   'PlayerAge',\n",
    "                   'PlayerWeight',\n",
    "                   'PlayerHeight', \n",
    "                   'Force',\n",
    "                   'JerseyNumber']\n",
    "\n",
    "    X_df = df.copy()\n",
    "    # Add features\n",
    "    if fe_phys:\n",
    "        X_df = pd.concat([X_df, phys_features], axis=1)\n",
    "\n",
    "    X_df = pd.concat([X_df, rusher_stats_features], axis=1)\n",
    "    X_df.drop(['rusher_id'], axis=1, inplace=True)\n",
    "    # Pospreprocesing\n",
    "\n",
    "    X_df = X_df.replace([np.inf, -np.inf], np.nan)\n",
    "\n",
    "    if fillna:\n",
    "        X_df.fillna(-999, inplace=True)\n",
    "    \n",
    "    if X_df.shape[0] > 1000:\n",
    "        X_df = reduce_mem_usage(X_df, verbose=verbose)\n",
    "\n",
    "    assert X_df.shape[0] == source_play_id.drop_duplicates().count()\n",
    "    return X_df, rusher_stats_counters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('/kaggle/input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'})\n",
    "# df_train = pd.read_csv('data/train.csv', dtype={'WindSpeed': 'object'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_preprocessed = preprocess_features(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropped cols: ['Position', 'Orientation', 'TimeSnap', 'GameWeather', 'PlayId', 'Stadium', 'HomeOnOffense', 'TimeHandoff', 'PlayerBirthDate', 'IsQB', 'Temperature', 'Humidity', 'Y', 'Yards', 'Team', 'IsOnOffense', 'WindSpeed', 'HomeTeamAbbr', 'ToLeft', 'DisplayName', 'FieldPosition', 'PlayerCollegeName', 'NflIdRusher', 'PlayDirection', 'Dis', 'NflId', 'IsRusher', 'Location', 'TeamOnOffense', 'VisitorTeamAbbr', 'X', 'GameId', 'PossessionTeam']\n",
      "Mem. usage decreased to 1.6352214813232422 Mb (78.55072463768116% reduction)\n"
     ]
    }
   ],
   "source": [
    "X_train, stats_counters = make_x(df_train_preprocessed, fillna=True, fe_phys=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = make_y(df_train_preprocessed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((23171, 51), (23171, 199))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, y_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regresssion ~magic~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.neighbors import KernelDensity\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "\n",
    "class RegressorConditional:\n",
    "    def get_o_cat(self, o):\n",
    "        return np.sum([o>pct for pct in self.percentiles], axis=0)\n",
    "    def __init__(self, model=ExtraTreesRegressor(\n",
    "        n_estimators=500, n_jobs=-1, bootstrap=True, oob_score=True)):\n",
    "        self.model = model\n",
    "    def fit(self, X, y):\n",
    "        targ = np.where(y>=0, np.log(1+np.abs(y)), -np.log(1+np.abs(y)))\n",
    "        self.model.fit(X, targ)\n",
    "        o = self.model.oob_prediction_\n",
    "        self.percentiles = np.percentile(o, list(range(10, 100, 10)))\n",
    "        o_cat = self.get_o_cat(o)\n",
    "        self.dist = {}\n",
    "        for oc in range(len(self.percentiles) + 1):\n",
    "            filt = [oi==oc for oi in o_cat]\n",
    "            kde = KernelDensity(kernel='exponential', metric='manhattan', bandwidth=0.3)\n",
    "            kde.fit(list(zip(y[filt])))\n",
    "            self.dist[oc] = np.exp(kde.score_samples(list(zip(range(-99, 100)))))\n",
    "            self.dist[oc] /= sum(self.dist[oc])\n",
    "    def predict_proba(self, X):\n",
    "        o = self.model.predict(X)\n",
    "        o_cat = self.get_o_cat(o)\n",
    "        return np.array([self.dist[oc] for oc in o_cat])\n",
    "\n",
    "y_train_yard_num = cdf_to_yard(y_train)\n",
    "reg = RegressorConditional()\n",
    "reg.fit(X_train.fillna(-999), y_train_yard_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pred_cdf(reg, X):\n",
    "    pred = np.cumsum(reg.predict_proba(X.fillna(-999)), axis=1)\n",
    "    pred = np.where(pred < 1e-4, 0, pred)\n",
    "    pred = np.clip(pred, 0, 1)\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SUBMIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def get_predictions(df_test, y_test, reg):\n",
    "    X, _ = make_x(preprocess_features(df_test, verbose=False), stats_counters=stats_counters, fe_phys=False, fillna=True, verbose=False)\n",
    "    y_pred = get_pred_cdf(reg, X)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = nflrush.make_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "iter_test = env.iter_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3438it [22:53,  2.50it/s]\n"
     ]
    }
   ],
   "source": [
    "for df_test, y_cdf_test in tqdm.tqdm(iter_test):\n",
    "    y_pred = get_predictions(df_test, y_cdf_test, reg)\n",
    "    env.predict(pd.DataFrame(data=y_pred, columns=y_cdf_test.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your submission file has been saved!  Once you `Commit` your Notebook and it finishes running, you can submit the file to the competition from the Notebook Viewer `Output` tab.\n"
     ]
    }
   ],
   "source": [
    "env.write_submission_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}