Single_Model_LGBM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction\n",
    "refer to this solution\n",
    "https://www.kaggle.com/pranav84/lightgbm-fixing-unbalanced-data-lb-0-9680/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import time\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split  \n",
    "import lightgbm as lgb\n",
    "import gc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jinpwa/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py:6201: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  sort=sort)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(58790469, 8)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load data\n",
    "dtypes = {\n",
    "        'ip'            : 'uint32',\n",
    "        'app'           : 'uint16',\n",
    "        'device'        : 'uint16',\n",
    "        'os'            : 'uint16',\n",
    "        'channel'       : 'uint16',\n",
    "        'is_attributed' : 'uint8',\n",
    "        'click_id'      : 'uint32'\n",
    "        }\n",
    "\n",
    "train_df = pd.read_csv(\"./data/train.csv\", skiprows=range(1,144903891), dtype=dtypes, nrows=40000000, \n",
    "                        usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])\n",
    "\n",
    "test_df = pd.read_csv(\"./data/test.csv\", dtype=dtypes, \n",
    "                       usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'])\n",
    "\n",
    "len_train = len(train_df)\n",
    "train_df=train_df.append(test_df)\n",
    "print(train_df.shape)\n",
    "\n",
    "del test_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add Feature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['hour'] = pd.to_datetime(train_df.click_time).dt.hour.astype('uint8')\n",
    "train_df['day'] = pd.to_datetime(train_df.click_time).dt.day.astype('uint8')\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Group Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# group ip-app\n",
    "gp = train_df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})\n",
    "train_df = train_df.merge(gp, on=['ip','app'], how='left')\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# group ip-day-hour\n",
    "gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_tcount'})\n",
    "train_df = train_df.merge(gp, on=['ip','day','hour'], how='left')\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grouping by ip-app-os combination...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# group ip-day-hour\n",
    "print('grouping by ip-app-os combination...')\n",
    "gp = train_df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})\n",
    "train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grouping by : ip_day_chl_var_hour\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('grouping by : ip_day_chl_var_hour')\n",
    "gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})\n",
    "train_df = train_df.merge(gp, on=['ip','day','channel'], how='left')\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grouping by : ip_app_os_var_hour\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('grouping by : ip_app_os_var_hour')\n",
    "gp = train_df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})\n",
    "train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')\n",
    "del gp\n",
    "gc.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grouping by : ip_app_channel_var_day\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('grouping by : ip_app_channel_var_day')\n",
    "gp = train_df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})\n",
    "train_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grouping by : ip_app_chl_mean_hour\n",
      "merging...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "70"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('grouping by : ip_app_chl_mean_hour')\n",
    "gp = train_df[['ip','app', 'channel','hour']].groupby(by=['ip', 'app', 'channel'])[['hour']].mean().reset_index().rename(index=str, columns={'hour': 'ip_app_channel_mean_hour'})\n",
    "print(\"merging...\")\n",
    "train_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')\n",
    "del gp\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 58790469 entries, 0 to 58790468\n",
      "Data columns (total 17 columns):\n",
      "app                         uint16\n",
      "channel                     uint16\n",
      "click_id                    float64\n",
      "click_time                  object\n",
      "device                      uint16\n",
      "ip                          uint32\n",
      "is_attributed               float64\n",
      "os                          uint16\n",
      "hour                        uint8\n",
      "day                         uint8\n",
      "ip_app_count                int64\n",
      "ip_tcount                   int64\n",
      "ip_app_os_count             int64\n",
      "ip_tchan_count              float64\n",
      "ip_app_os_var               float64\n",
      "ip_app_channel_var_day      float64\n",
      "ip_app_channel_mean_hour    float64\n",
      "dtypes: float64(6), int64(3), object(1), uint16(4), uint32(1), uint8(2)\n",
      "memory usage: 5.6+ GB\n"
     ]
    }
   ],
   "source": [
    "train_df.info()\n",
    "train_df['ip_app_count'] = train_df['ip_app_count'].astype('uint16')\n",
    "train_df['ip_tcount'] = train_df['ip_tcount'].astype('uint16')\n",
    "train_df['ip_app_os_count'] = train_df['ip_app_os_count'].astype('uint16')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size:  37500000\n",
      "valid size:  2500000\n",
      "test size :  18790469\n"
     ]
    }
   ],
   "source": [
    "# data split \n",
    "test_df = train_df[len_train:]\n",
    "val_df = train_df[(len_train-2500000):len_train]\n",
    "train_df = train_df[:(len_train-2500000)]\n",
    "\n",
    "print(\"train size: \", len(train_df))\n",
    "print(\"valid size: \", len(val_df))\n",
    "print(\"test size : \", len(test_df))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = 'is_attributed'\n",
    "predictors = ['app','device','os', 'channel', 'hour', 'day', \n",
    "              'ip_tcount', 'ip_tchan_count', 'ip_app_count',\n",
    "              'ip_app_os_count', 'ip_app_os_var',\n",
    "              'ip_app_channel_var_day','ip_app_channel_mean_hour']\n",
    "categorical = ['app', 'device', 'os', 'channel', 'hour', 'day']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lgb_modelfit_nocv(params, dtrain, dvalid, predictors, target='target', objective='binary', metrics='auc',\n",
    "                 feval=None, early_stopping_rounds=20, num_boost_round=3000, verbose_eval=10, categorical_features=None):\n",
    "    lgb_params = {\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': objective,\n",
    "        'metric':metrics,\n",
    "        'learning_rate': 0.01,\n",
    "        #'is_unbalance': 'true',  #because training data is unbalance (replaced with scale_pos_weight)\n",
    "        'num_leaves': 31,  # we should let it be smaller than 2^(max_depth)\n",
    "        'max_depth': -1,  # -1 means no limit\n",
    "        'min_child_samples': 20,  # Minimum number of data need in a child(min_data_in_leaf)\n",
    "        'max_bin': 255,  # Number of bucketed bin for feature values\n",
    "        'subsample': 0.6,  # Subsample ratio of the training instance.\n",
    "        'subsample_freq': 0,  # frequence of subsample, <=0 means no enable\n",
    "        'colsample_bytree': 0.3,  # Subsample ratio of columns when constructing each tree.\n",
    "        'min_child_weight': 5,  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n",
    "        'subsample_for_bin': 200000,  # Number of samples for constructing bin\n",
    "        'min_split_gain': 0,  # lambda_l1, lambda_l2 and min_gain_to_split to regularization\n",
    "        'reg_alpha': 0,  # L1 regularization term on weights\n",
    "        'reg_lambda': 0,  # L2 regularization term on weights\n",
    "        'nthread': 4,\n",
    "        'verbose': 0,\n",
    "        'metric':metrics\n",
    "    }\n",
    "\n",
    "    lgb_params.update(params)\n",
    "\n",
    "    print(\"preparing validation datasets\")\n",
    "\n",
    "    xgtrain = lgb.Dataset(dtrain[predictors].values, label=dtrain[target].values,\n",
    "                          feature_name=predictors,\n",
    "                          categorical_feature=categorical_features\n",
    "                          )\n",
    "    xgvalid = lgb.Dataset(dvalid[predictors].values, label=dvalid[target].values,\n",
    "                          feature_name=predictors,\n",
    "                          categorical_feature=categorical_features\n",
    "                          )\n",
    "\n",
    "    evals_results = {}\n",
    "\n",
    "    bst1 = lgb.train(lgb_params, \n",
    "                     xgtrain, \n",
    "                     valid_sets=[xgtrain, xgvalid], \n",
    "                     valid_names=['train','valid'], \n",
    "                     evals_result=evals_results, \n",
    "                     num_boost_round=num_boost_round,\n",
    "                     early_stopping_rounds=early_stopping_rounds,\n",
    "                     verbose_eval=10, \n",
    "                     feval=feval)\n",
    "\n",
    "    n_estimators = bst1.best_iteration\n",
    "    print(\"\\nModel Report\")\n",
    "    print(\"n_estimators : \", n_estimators)\n",
    "    print(metrics+\":\", evals_results['valid'][metrics][n_estimators-1])\n",
    "\n",
    "    return bst1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preparing validation datasets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jinpwa/anaconda3/lib/python3.6/site-packages/lightgbm/basic.py:1205: UserWarning: Using categorical_feature in Dataset.\n",
      "  warnings.warn('Using categorical_feature in Dataset.')\n",
      "/home/jinpwa/anaconda3/lib/python3.6/site-packages/lightgbm/basic.py:762: UserWarning: categorical_feature in param dict is overridden.\n",
      "  warnings.warn('categorical_feature in param dict is overridden.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 100 rounds.\n",
      "[10]\ttrain's auc: 0.89076\tvalid's auc: 0.918921\n",
      "[20]\ttrain's auc: 0.899339\tvalid's auc: 0.927102\n",
      "[30]\ttrain's auc: 0.857662\tvalid's auc: 0.816904\n",
      "[40]\ttrain's auc: 0.865027\tvalid's auc: 0.84443\n",
      "[50]\ttrain's auc: 0.89131\tvalid's auc: 0.908471\n",
      "[60]\ttrain's auc: 0.883883\tvalid's auc: 0.904787\n",
      "[70]\ttrain's auc: 0.883976\tvalid's auc: 0.904768\n",
      "[80]\ttrain's auc: 0.884013\tvalid's auc: 0.904797\n",
      "[90]\ttrain's auc: 0.884093\tvalid's auc: 0.904939\n",
      "[100]\ttrain's auc: 0.91594\tvalid's auc: 0.934895\n",
      "Early stopping, best iteration is:\n",
      "[1]\ttrain's auc: 0.946506\tvalid's auc: 0.970916\n",
      "\n",
      "Model Report\n",
      "n_estimators :  1\n",
      "auc: 0.9709156663984586\n",
      "[415.3159923553467]: model training time\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "params = {\n",
    "    'learning_rate': 0.15,\n",
    "    'is_unbalance': 'true', # replaced with scale_pos_weight argument\n",
    "    'num_leaves': 7,  # 2^max_depth - 1\n",
    "    'max_depth': 3,  # -1 means no limit\n",
    "    'min_child_samples': 100,  # Minimum number of data need in a child(min_data_in_leaf)\n",
    "    'max_bin': 100,  # Number of bucketed bin for feature values\n",
    "    'subsample': 0.7,  # Subsample ratio of the training instance.\n",
    "    'subsample_freq': 1,  # frequence of subsample, <=0 means no enable\n",
    "    'colsample_bytree': 0.9,  # Subsample ratio of columns when constructing each tree.\n",
    "    'min_child_weight': 0  # Minimum sum of instance weight(hessian) needed in a child(leaf)\n",
    "    #'scale_pos_weight':9 # because training data is extremely unbalanced \n",
    "}\n",
    "\n",
    "bst = lgb_modelfit_nocv(params, \n",
    "                        train_df, \n",
    "                        val_df, \n",
    "                        predictors, \n",
    "                        target, \n",
    "                        objective='binary', \n",
    "                        metrics='auc',\n",
    "                        early_stopping_rounds=100, \n",
    "                        verbose_eval=True, \n",
    "                        num_boost_round=500, \n",
    "                        categorical_features=categorical)\n",
    "\n",
    "print('[{}]: model training time'.format(time.time() - start_time))\n",
    "#del train_df\n",
    "#del val_df\n",
    "#gc.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "writing...\n",
      "done...\n"
     ]
    }
   ],
   "source": [
    "sub = pd.DataFrame()\n",
    "sub['click_id'] = test_df['click_id'].astype('int')\n",
    "sub['is_attributed'] = bst.predict(test_df[predictors])\n",
    "print(\"writing...\")\n",
    "sub.to_csv('sub_lgb_early100.csv',index=False)\n",
    "print(\"done...\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}