toolkit_x.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGBoost\n",
    "\n",
    "param_grid={\n",
    "    'max_depth':[4],\n",
    "    'learning_rate':[0.1, 0.2, 0.3],\n",
    "    'gamma':[0],\n",
    "    'reg_lambda':[1.0],\n",
    "    'scale_pos_weight': [1]\n",
    "}\n",
    "\n",
    "optimal_params = GridSearchCV(\n",
    "                        estimator=xgb.XGBClassifier(\n",
    "                            objective='binary:logistic'\n",
    "                            , tree_method='gpu_hist'\n",
    "                            , subsample=0.8\n",
    "                            , colsample_bytree=0.8\n",
    "                            , seed=42\n",
    "                            , use_label_encoder=False)\n",
    "                        ,param_grid=param_grid\n",
    "                        ,scoring='roc_auc'\n",
    "                        ,verbose=0\n",
    "                        ,n_jobs=-1\n",
    "                        ,cv=3\n",
    "                        )\n",
    "\n",
    "optimal_params.fit(X_train,\n",
    "           y_train,\n",
    "           verbose=False,\n",
    "           early_stopping_rounds=10,\n",
    "           eval_metric='auc',\n",
    "           eval_set=[(X_test, y_test)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random Forest\n",
    "\n",
    "criterion =['gini', 'entropy']\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10, 15]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 5, 10, 15]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "\n",
    "param_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap,\n",
    "               'criterion': criterion}\n",
    "\n",
    "optimal_params = RandomizedSearchCV(\n",
    "                                estimator = RandomForestClassifier()\n",
    "                                , param_distributions = param_grid\n",
    "                                , n_iter = 100\n",
    "                                , cv = 3\n",
    "                                , verbose=2\n",
    "                                , random_state=42\n",
    "                                , n_jobs = -1)\n",
    "\n",
    "optimal_params.fit(X_processed, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = [{'kernel' : ('linear', 'poly', 'rbf', 'sigmoid'),\n",
    "         'C' : [1,5,10],\n",
    "         'degree' : [3,8],\n",
    "         'coef0' : [0.01,10,0.5],\n",
    "         'gamma' : ('auto','scale')\n",
    "              }]\n",
    "\n",
    "param_grid = [{\n",
    "    'C': [0.5, 1, 10, 100],\n",
    "    'gamma': ['scale', 1, 0.1, 0.01, 0.001, 0.0001],\n",
    "    'kernel': ['rbf'],\n",
    "}]\n",
    "\n",
    "optimal_params = GridSearchCV(\n",
    "                            SVC(),\n",
    "                            param_grid,\n",
    "                            cv=5,\n",
    "                            scoring='acccuracy'\n",
    "                            )\n",
    "\n",
    "optimal_params.fit(X_train, y_train)\n",
    "print(optimal_params.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LightGBM using hyperopt\n",
    "\n",
    "def objective_lgb(space):\n",
    "    model=lgb.LGBMClassifier(\n",
    "        n_estimators=int(space['n_estimators']),\n",
    "        max_depth=int(space['max_depth']),\n",
    "        num_leaves=int(space['num_leaves']),\n",
    "        learning_rate=space['learning_rate'],\n",
    "        gamma=space['gamma'],\n",
    "        reg_alpha=space['reg_alpha'],\n",
    "        reg_lambda=space['reg_lambda'],\n",
    "        colsample_bytree=space['colsample_bytree'],\n",
    "        min_child_weight=space['min_child_weight'],\n",
    "        boosting_type=space['boosting_type']\n",
    "    )\n",
    "    \n",
    "    score = -cross_val_score(model, x_train, y_train, cv=10, scoring='roc_auc').mean()\n",
    "    \n",
    "    return score\n",
    "    \n",
    "    \n",
    "space= {\n",
    "    'n_estimators': hp.quniform('n_estimators', 5, 35, 1),\n",
    "    'max_depth': hp.quniform('max_depth', 5, 15, 1),\n",
    "    'learning_rate': hp.loguniform('learning_rate', np.log(0.01), np.log(1)),\n",
    "    'gamma': hp.quniform('gamma', 0.01, 0.1, 0.05),\n",
    "    'num_leaves': hp.quniform('num_leaves', 5, 50, 1),\n",
    "    'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),\n",
    "    'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),\n",
    "    'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0),\n",
    "    'min_child_weight': hp.quniform('min_child_weight', 0, 10, 1),\n",
    "    'boosting_type': hp.choice('boosting_type', ['gbdt', 'dart'])   \n",
    "}\n",
    "\n",
    "trials = Trials()\n",
    "\n",
    "best_param = fmin(fn=objective_lgb, \n",
    "                  space=space, \n",
    "                  algo=tpe.suggest, \n",
    "                  max_evals=100, \n",
    "                  trials=trials, \n",
    "                  rstate= np.random.RandomState(1)\n",
    "                 )\n",
    "\n",
    "print(best_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import norm, skew \n",
    "\n",
    "sns.set_style(\"whitegrid\")\n",
    "plt.figure(figsize=(12,4))\n",
    "\n",
    "sns.distplot(housing_df.SalePrice, fit=norm);\n",
    "(mu, sigma) = norm.fit(housing_df.SalePrice)\n",
    "print( '\\n mu = {:.2f} and sigma = {:.2f}\\n'.format(mu, sigma))\n",
    "plt.legend(['Normal dist. ($\\mu=$ {:.2f} and $\\sigma=$ {:.2f} )'.format(mu, sigma)],\n",
    "            loc='upper right')\n",
    "\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('SalePrice distribution')\n",
    "\n",
    "# calculate skewness of data\n",
    "numeric_feats = data.dtypes[data.dtypes != 'object'].index\n",
    "skewed_feats = data[numeric_feats].apply(lambda x: skew(x)).sort_values(ascending=False)\n",
    "high_skew = skewed_feats[abs(skewed_feats) > 0.5]\n",
    "high_skew"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_features=[]\n",
    "for c in train_df.columns:\n",
    "    col_type = train_df[c].dtype\n",
    "    if col_type == 'object' or col_type.name == 'category':\n",
    "        train_df[c] = train_df[c].astype('category')\n",
    "        categorical_features.append(c)\n",
    "print (categorical_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cat_feat in categorical_feat:\n",
    "    data[cat_feat] = data[cat_feat].fillna(str(data[cat_feat][:len(train_df)].value_counts().index[0]))\n",
    "    \n",
    "for num_feet in numerical_feat:\n",
    "    data[num_feet] = data[num_feet].fillna(data[num_feet][:len(train_df)].mean())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}