parameter_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "import first\n",
    "import pandas as pd\n",
    "import data_io as di\n",
    "from sklearn import cross_validation, metrics\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.grid_search import GridSearchCV \n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline\n",
    "from matplotlib.pylab import rcParams\n",
    "rcParams['figure.figsize'] = 12, 4\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X, y,useTrainCV=True, cv_folds=5, early_stopping_rounds=50):\n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgtrain = xgb.DMatrix(X, y)\n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], nfold=cv_folds,\n",
    "            metrics='auc', early_stopping_rounds=early_stopping_rounds)\n",
    "        alg.set_params(n_estimators=cvresult.shape[0])\n",
    "\n",
    "#Fit the algorithm on the data\n",
    "    alg.fit(X, y,eval_metric='error')\n",
    "\n",
    "#Predict training set:\n",
    "    dtrain_predictions = alg.predict(X)\n",
    "    dtrain_predprob = alg.predict_proba(X)[:,1]\n",
    "\n",
    "#Print model report:\n",
    "    print(\"\\nModel Report\")\n",
    "    print(\"Accuracy : %.4g\" % metrics.accuracy_score(y, dtrain_predictions))\n",
    "    print(\"AUC Score (Train): %f\" % metrics.roc_auc_score(y, dtrain_predprob))\n",
    "\n",
    "    feat_imp = pd.Series(alg.feature_importances_).sort_values(ascending=False)\n",
    "    feat_imp.plot(kind='bar', title='Feature Importances')\n",
    "    plt.ylabel('Feature Importance Score')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1 = xgb.XGBClassifier(\n",
    " learning_rate =0.1,\n",
    " n_estimators=200,\n",
    " max_depth=5,\n",
    " min_child_weight=1,\n",
    " gamma=0,\n",
    " subsample=0.8,\n",
    " colsample_bytree=0.8,\n",
    " objective= 'binary:logistic',\n",
    " scale_pos_weight=1,\n",
    " random_state=27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.9344\n",
      "AUC Score (Train): 0.934377\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAEFCAYAAADdbkBUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHQNJREFUeJzt3Xu4XXV95/H3hyDIHZWAEIhBxSJaoE4Eq0wrRZSLGh+v\n4G3EC+JA0aejQ8axrbda7KN2dNRitDh4QRxRNJYAonWwXqiJiFwUNEI0SZGAXBIDBUK+88dex26O\n57JOOOvs7OP79Tz7OXv91mV/sk4g3/M7v/X7paqQJEmSNL22GXQASZIkaTay0JYkSZI6YKEtSZIk\ndcBCW5IkSeqAhbYkSZLUAQttSZIkqQMW2pIkSVIHLLQlqZFkVZK7k/ym77XPg7zm05Osma6MLT/z\n/yR590x+5niSvD3JZwadQ5IGwUJbkh7oOVW1c9/r3wYZJsm2g/z8B2OYs0vSdLDQlqQWkjwlyXeT\n3JHkR0me3rfvpCQ/SbIhyQ1JXt+07wRcBOzT30M+usd5dK9307N+RpKrgI1Jtm3O+2KSW5LcmOT0\nlrkXJKkm4+oktyc5JcmTk1zV/Hk+3Hf8q5J8J8mHk9yZ5LokR/Xt3yfJ0iS3JVmZ5HV9+96e5Pwk\nn0myHjgFeCvwkubP/qOJ7lf/vUjy35KsS3JTkpP69u+Q5P1JftHk+3aSHVp8j17VfNaG5v69rM39\nk6QHw94GSZpEknnAhcArgIuBo4AvJjmwqm4B1gHPBm4A/gS4KMnyqroiybHAZ6pq377rtfnYE4Hj\ngVuBzcBXga807fsCX09yfVVd0vKPcThwQJNvafPneAbwEOCHSb5QVZf1HXs+sAfwfOBLSfavqtuA\n84BrgH2AA4FLk/y8qv65OXcR8CLglcD2zTUeW1Uv78sy7v1q9j8S2A2YBxwNnJ/ky1V1O/A+4AnA\nU4FfNVk3T/Q9Au4CPgQ8uaquT7I38PCW902Stpg92pL0QF9uekTvSPLlpu3lwLKqWlZVm6vqUmAF\ncBxAVV1YVT+vnsuArwH/+UHm+FBVra6qu4EnA3Or6p1VdW9V3QB8HDhhCtd7V1X9e1V9DdgIfK6q\n1lXVWuBfgD/qO3Yd8L+q6r6q+jxwPXB8kv2ApwFnNNe6EvgEvaJ6xPeq6svNfbp7rCAt7td9wDub\nz18G/Ab4gyTbAK8G3lhVa6vq/qr6blXdwyTfI3o/rDwxyQ5VdVNVXTuFeydJW8RCW5Ie6HlVtXvz\nel7T9ijgRX0F+B3AEcDeAEmOTXJ5M5ziDnrF3R4PMsfqvvePojf8pP/z3wrsNYXr3dz3/u4xtnfu\n215bVdW3/Qt6Pdj7ALdV1YZR++aNk3tMLe7Xr6tqU9/2XU2+PYCHAj8f47Ljfo+qaiPwEnpDWW5K\ncmHT0y1JnbLQlqTJrQY+3VeA715VO1XVmUm2B75Ib0jDXlW1O7AMGBkfUmNcbyOwY9/2I8c4pv+8\n1cCNoz5/l6o6bozzpsO8PHB8y3zg35rXw5PsMmrf2nFy/852i/s1kVuBfwceM8a+cb9HAFV1SVUd\nTe+Ho+vo/UZAkjploS1Jk/sM8Jwkz0oyJ8lDm4f29gW2ozcW+RZgUzMm+5l9594MPCLJbn1tVwLH\nJXl4kkcCb5rk878PbGgekNyhyfDEJE+etj/hA+0JnJ7kIUleBDye3rCM1cB3gb9t7sHBwGvo3Z/x\n3AwsaIZ9wOT3a1xVtRk4G/hA81DmnCR/3BTv436PkuyVZFF6D6feQ28oyuYp3hNJmjILbUmaRFNg\nLqI3XOMWer2nbwG2aYZRnA78X+B24KX0HjYcOfc64HPADc2Qhn2ATwM/AlbRG5/8+Uk+/356Dw8e\nCtxIr2f3E/QeGOzCv9J7cPJW4G+AF1bVr5t9JwIL6PVuXwD8dVV9fYJrfaH5+uskV0x2v1p4M3A1\nsBy4DXgvve/DuN+j5vUXTebbgD8F3jCFz5SkLZIHDsOTJP0+S/Iq4LVVdcSgs0jSsLNHW5IkSeqA\nhbYkSZLUgU6HjiQ5BvggMAf4xMjT3337XwacQe9p8w3AG6pqZOWwVU3b/cCmqlrYWVBJkiRpmnVW\naCeZA/yU3qpea+g9uHJiVf2475inAj+pqtubJ8/fXlWHN/tWAQur6tZOAkqSJEkd6nLoyGHAyqq6\noarupbds76L+A5oVvW5vNi+nt6ywJEmSNPS27fDa83jgCmFrgMMnOP41wEV92wV8Pcn9wMeqaslk\nH7jHHnvUggULtiCqJEmS1M4PfvCDW6tq7mTHdVlot5bkSHqFdv90UkdU1dokewKXJrmuqr41xrkn\nAycDzJ8/nxUrVsxIZkmSJP1+SvKLNsd1OXRkLbBf3/a+PHCZXgCalcU+ASzqWxCBqlrbfF1Hb1GE\nw8b6kKpaUlULq2rh3LmT/mAhSZIkzYguC+3lwAFJ9k+yHXACo1b/SjIf+BLwiqr6aV/7Tkl2GXlP\nb3neazrMKkmSJE2rzoaOVNWmJKcBl9Cb3u/sqro2ySnN/rOAvwIeAXw0CfzHNH57ARc0bdsC51bV\nxV1llSRJkqbbrFqCfeHCheUYbUmSJHUpyQ/arPHiypCSJElSByy0JUmSpA5YaEuSJEkdsNCWJEmS\nOrBVLFizNVuw+MJBR5jUqjOPH3QESZIkjWKPtiRJktQBC21JkiSpAxbakiRJUgcstCVJkqQOWGhL\nkiRJHbDQliRJkjrg9H6aMcMwVSI4XaIkSZoe9mhLkiRJHbDQliRJkjpgoS1JkiR1wEJbkiRJ6oCF\ntiRJktQBC21JkiSpAxbakiRJUgcstCVJkqQOWGhLkiRJHbDQliRJkjpgoS1JkiR1wEJbkiRJ6oCF\ntiRJktQBC21JkiSpAxbakiRJUgcstCVJkqQOWGhLkiRJHbDQliRJkjpgoS1JkiR1wEJbkiRJ6oCF\ntiRJktQBC21JkiSpAxbakiRJUgc6LbSTHJPk+iQrkyweY//LklyV5Ook301ySNtzJUmSpK1ZZ4V2\nkjnAR4BjgYOAE5McNOqwG4E/rao/BN4FLJnCuZIkSdJWq8se7cOAlVV1Q1XdC5wHLOo/oKq+W1W3\nN5uXA/u2PVeSJEnamnVZaM8DVvdtr2naxvMa4KItPFeSJEnaqmzb9sAkO1bVXV2ESHIkvUL7iC04\n92TgZID58+dPczJJkiRpy0zao53kqUl+DFzXbB+S5KMtrr0W2K9ve9+mbfT1DwY+ASyqql9P5VyA\nqlpSVQurauHcuXNbxJIkSZK612boyN8DzwJ+DVBVPwL+pMV5y4EDkuyfZDvgBGBp/wFJ5gNfAl5R\nVT+dyrmSJEnS1qzV0JGqWp2kv+n+FudsSnIacAkwBzi7qq5Nckqz/yzgr4BHAB9trr+p6Z0e89wp\n/LkkSZKkgWpTaK9O8lSgkjwEeCPwkzYXr6plwLJRbWf1vX8t8Nq250qSJEnDos3QkVOAU+nN+rEW\nOLTZliRJkjSOCXu0m4VjXlFVL5uhPJIkSdKsMGGPdlXdD7x0hrJIkiRJs0abMdrfTvJh4PPAxpHG\nqrqis1SSJEnSkGtTaB/afH1nX1sBfzb9cSRJkqTZYdJCu6qOnIkgkiRJ0mzSZmXI3ZJ8IMmK5vX+\nJLvNRDhJkiRpWLWZ3u9sYAPw4ua1Hvhkl6EkSZKkYddmjPZjquoFfdvvSHJlV4EkSZKk2aBNj/bd\nSY4Y2UjyNODu7iJJkiRJw69Nj/YbgHP6xmXfDryqs0SSJEnSLNBm1pErgUOS7Npsr+88lSRJkjTk\n2sw68p4ku1fV+qpan+RhSd49E+EkSZKkYdVmjPaxVXXHyEZV3Q4c110kSZIkafi1KbTnJNl+ZCPJ\nDsD2ExwvSZIk/d5r8zDkZ4FvJBmZO/sk4JzuIkmSJEnDr83DkO9N8iPgGUAB76qqSzpPJkmSJA2x\nNj3aVNXFSZYDfwLc2m0kSZIkafiNO0Y7yT8leWLzfm/gGuDVwKeTvGmG8kmSJElDaaKHIfevqmua\n9ycBl1bVc4DD6RXckiRJksYxUaF9X9/7o4BlAFW1AdjcZShJkiRp2E00Rnt1kj8H1gBPAi6G307v\n95AZyCZJkiQNrYl6tF8DPAF4FfCSvkVrngJ8cryTJEmSJE3Qo11V64BTxmj/JvDNLkNJkiRJw67N\nypCSJEmSpshCW5IkSeqAhbYkSZLUgUkL7SSPS/KNJNc02wcneVv30SRJkqTh1aZH++PA/6CZV7uq\nrgJO6DKUJEmSNOzaFNo7VtX3R7Vt6iKMJEmSNFu0KbRvTfIYoACSvBC4qdNUkiRJ0pCbaGXIEacC\nS4ADk6wFbgRe3mkqSZIkachNWmhX1Q3AM5LsBGxTVRu6jyVJkiQNtzazjrwnye5VtbGqNiR5WJJ3\nz0Q4SZIkaVi1GaN9bFXdMbJRVbcDx7W5eJJjklyfZGWSxWPsPzDJ95Lck+TNo/atSnJ1kiuTrGjz\neZIkSdLWos0Y7TlJtq+qewCS7ABsP9lJSeYAHwGOBtYAy5Msraof9x12G3A68LxxLnNkVd3aIqMk\nSZK0VWlTaH8W+EaSTzbbJwHntDjvMGBlM8abJOcBi4DfFtpVtQ5Yl+T4KaWWJEmStnJtHoZ8b5Kr\ngKOapndV1SUtrj0PWN23vQY4fArZCvh6kvuBj1XVkimcK0mSJA1Umx5tquoi4KKOs4x2RFWtTbIn\ncGmS66rqW6MPSnIycDLA/PnzZziiJEmSNLY2s448P8nPktyZZH2SDUnWt7j2WmC/vu19m7ZWqmpt\n83UdcAG9oShjHbekqhZW1cK5c+e2vbwkSZLUqTazjvwd8Nyq2q2qdq2qXapq1xbnLQcOSLJ/ku2A\nE4ClbUIl2SnJLiPvgWcC17Q5V5IkSdoatBk6cnNV/WSqF66qTUlOAy4B5gBnV9W1SU5p9p+V5JHA\nCmBXYHOSNwEHAXsAFyQZyXhuVV081QySJEnSoLQptFck+TzwZeCekcaq+tJkJ1bVMmDZqLaz+t7/\nit6QktHWA4e0yCZJkiRtldoU2rsCd9EbvjGigEkLbUmSJOn3VZvp/U6aiSCSJEnSbDJpoZ3kocBr\ngCcADx1pr6pXd5hLkiRJGmptZh35NPBI4FnAZfTGVG/oMpQkSZI07NoU2o+tqr8ENlbVOcDxTG2F\nR0mSJOn3TptC+77m6x1JngjsBuzZXSRJkiRp+LWZdWRJkocBb6O34MzOwF92mkqSJEkacm0K7W9U\n1e3At4BHAyTZv9NUkiRJ0pBrM3Tki2O0nT/dQSRJkqTZZNwe7SQH0pvSb7ckz+/btSt90/xJkiRJ\n+l0TDR35A+DZwO7Ac/raNwCv6zKUJEmSNOzGLbSr6itJ/gk4o6reM4OZJEmSpKE34cOQVXV/kucB\nFtrSVmbB4gsHHaGVVWceP+gIkiQNRJtZR76T5MPA54GNI41VdUVnqSRJkqQh16bQPrT5+s6+tgL+\nbPrjSJIkSbPDpIV2VR05E0EkSZKk2WTSebST7JbkA0lWNK/3J9ltJsJJkiRJw6rNgjVn05vS78XN\naz3wyS5DSZIkScOuzRjtx1TVC/q235Hkyq4CSZIkSbNBmx7tu5McMbKR5GnA3d1FkiRJkoZfmx7t\nNwDnNOOyA9wG/JdOU0mSJElDrs2sI1cChyTZtdle33kqSZIkaci1mXXkEUk+BPw/4JtJPpjkEZ0n\nkyRJkoZYmzHa5wG3AC8AXti8/3yXoSRJkqRh12aM9t5V9a6+7XcneUlXgSRJkqTZoE2P9teSnJBk\nm+b1YuCSroNJkiRJw6xNof064Fzg3uZ1HvD6JBuS+GCkJEmSNIY2s47sMhNBJEmSpNmkzRhtkhwM\nLOg/vqq+1FEmSZIkaehNWmgnORs4GLgW2Nw0F2ChLUmSJI2jTY/2U6rqoM6TSNIALVh84aAjTGrV\nmccPOoIkaQraPAz5vSQW2pIkSdIUtOnR/hS9YvtXwD1AgKqqgztNJkmSJA2xNoX2PwKvAK7mP8Zo\nS5IkSZpAm6Ejt1TV0qq6sap+MfJqc/EkxyS5PsnKJIvH2H9gku8luSfJm6dyriRJkrQ1a9Oj/cMk\n5wJfpTd0BJh8er8kc4CPAEcDa4DlSZZW1Y/7DrsNOB143hacK0mSJG212hTaO9ArsJ/Z19Zmer/D\ngJVVdQNAkvOARcBvi+WqWgesSzL6UfpJz5UkSZK2Zm1WhjxpC689D1jdt70GOHwGzpUkSZIGbtxC\nO8n/ptdzPaaqOr2TRFOU5GTgZID58+cPOI0kSZLUM1GP9ooHee21wH592/s2bdN6blUtAZYALFy4\ncNwfDCRJkqSZNG6hXVXnPMhrLwcOSLI/vSL5BOClM3CuJEmSNHBtHobcIlW1KclpwCXAHODsqro2\nySnN/rOSPJJez/muwOYkbwIOqqr1Y53bVVZJkiRpunVWaANU1TJg2ai2s/re/4resJBW50qSJEnD\nos2CNZIkSZKmaNJCO8njknwjyTXN9sFJ3tZ9NEmSJGl4tRk68nHgLcDHAKrqqmalyHd3GUySNJwW\nLL5w0BFaWXXm6LXSJGl6tRk6smNVfX9U26YuwkiSJEmzRZse7VuTPIZm8ZokLwRu6jSVJEkC/A2B\nNMzaFNqn0lsQ5sAka4EbgZd1mkqSJEkachMW2km2ARZW1TOS7ARsU1UbZiaaJEmSNLwmHKNdVZuB\n/96832iRLUmSJLXT5mHIryd5c5L9kjx85NV5MkmSJGmItRmj/ZLm66l9bQU8evrjSJIkSbPDpIV2\nVe0/E0EkSZKk2WTSQjvJK8dqr6pPTX8cSZIkaXZoM3TkyX3vHwocBVwBWGhLkiRJ42gzdOTP+7eT\n7A6c11kiSZIkaRZoM+vIaBsBx21LkiRJE2gzRvurNMuv0yvMDwK+0GUoSZIkadi1GaP9vr73m4Bf\nVNWajvJIkiR1ZsHiCwcdoZVVZx4/6AiaBm2GjhxXVZc1r+9U1Zok7+08mSRJkjTE2hTaR4/Rdux0\nB5EkSZJmk3GHjiR5A/BfgUcnuapv1y7Ad7oOJkmSJA2zicZonwtcBPwtsLivfUNV3dZpKkmSJGnI\njVtoV9WdwJ3AiQBJ9qS3YM3OSXauql/OTERJkiRp+Ew6RjvJc5L8DLgRuAxYRa+nW5IkSdI42jwM\n+W7gKcBPq2p/ekuwX95pKkmSJGnItSm076uqXwPbJNmmqr4JLOw4lyRJkjTU2ixYc0eSnYF/AT6b\nZB29ZdglSZIkjaNNj/Yi4C7gTcDFwM+B53QZSpIkSRp2k/ZoV9XGJI8CDqiqc5LsCMzpPpokSZI0\nvNrMOvI64HzgY03TPODLXYaSJEmShl2boSOnAk8D1gNU1c+APbsMJUmSJA27NoX2PVV178hGkm2B\n6i6SJEmSNPzaFNqXJXkrsEOSo4EvAF/tNpYkSZI03NoU2ouBW4CrgdcDy4C3dRlKkiRJGnbjzjqS\nZH5V/bKqNgMfb16SJEmSWpioR/u3M4sk+eKWXDzJMUmuT7IyyeIx9ifJh5r9VyV5Ut++VUmuTnJl\nkhVb8vmSJEnSoEw0j3b63j96qhdOMgf4CHA0sAZYnmRpVf2477BjgQOa1+HAPzRfRxxZVbdO9bMl\nSZKkQZuoR7vGed/WYcDKqrqhmbXkPHqrTPZbBHyqei4Hdk+y9xZ8liRJkrRVmajQPiTJ+iQbgIOb\n9+uTbEiyvsW15wGr+7bXNG1tjyng60l+kOTkFp8nSZIkbTXGHTpSVYNeZv2IqlqbZE/g0iTXVdW3\nRh/UFOEnA8yfP3+mM0qSJEljajO935ZaC+zXt71v09bqmKoa+boOuIDeUJTfUVVLqmphVS2cO3fu\nNEWXJEmSHpwuC+3lwAFJ9k+yHXACsHTUMUuBVzazjzwFuLOqbkqyU5JdAJLsBDwTuKbDrJIkSdK0\nmmjWkQelqjYlOQ24BJgDnF1V1yY5pdl/Fr3Fb44DVgJ3ASc1p+8FXJBkJOO5VXVxV1klSZKk6dZZ\noQ1QVcvoFdP9bWf1vS/g1DHOuwE4pMtskiRJUpe6HDoiSZIk/d6y0JYkSZI6YKEtSZIkdcBCW5Ik\nSeqAhbYkSZLUAQttSZIkqQMW2pIkSVIHLLQlSZKkDlhoS5IkSR2w0JYkSZI60OkS7JIkSZqdFiy+\ncNARWll15vED+2x7tCVJkqQOWGhLkiRJHbDQliRJkjpgoS1JkiR1wEJbkiRJ6oCFtiRJktQBC21J\nkiSpAxbakiRJUgcstCVJkqQOWGhLkiRJHbDQliRJkjpgoS1JkiR1wEJbkiRJ6oCFtiRJktQBC21J\nkiSpAxbakiRJUgcstCVJkqQOWGhLkiRJHbDQliRJkjpgoS1JkiR1wEJbkiRJ6oCFtiRJktQBC21J\nkiSpAxbakiRJUgc6LbSTHJPk+iQrkyweY3+SfKjZf1WSJ7U9V5IkSdqadVZoJ5kDfAQ4FjgIODHJ\nQaMOOxY4oHmdDPzDFM6VJEmStlpd9mgfBqysqhuq6l7gPGDRqGMWAZ+qnsuB3ZPs3fJcSZIkaau1\nbYfXnges7tteAxze4ph5Lc8FIMnJ9HrDAX6T5PoHkXkm7AHcOp0XzHun82pDx/s5vbyf02ta76f3\n0r+b08j7Ob28n9NnWO7lo9oc1GWhPSOqagmwZNA52kqyoqoWDjrHbOH9nF7ez+nl/Zw+3svp5f2c\nXt7P6TPb7mWXhfZaYL++7X2btjbHPKTFuZIkSdJWq8sx2suBA5Lsn2Q74ARg6ahjlgKvbGYfeQpw\nZ1Xd1PJcSZIkaavVWY92VW1KchpwCTAHOLuqrk1ySrP/LGAZcBywErgLOGmic7vKOsOGZpjLkPB+\nTi/v5/Tyfk4f7+X08n5OL+/n9JlV9zJVNegMkiRJ0qzjypCSJElSByy0JUmSpA5YaEuSJEkdsNDW\nUElyWJInN+8PSvIXSY4bdK7ZIMmnBp1hWCU5PMmuzfsdkrwjyVeTvDfJboPON4ySHJjkqCQ7j2o/\nZlCZhllzP89I8qHmdUaSxw8617BLckTz79AzB51lGCU5Pcl+kx85vHwYcoCSnFRVnxx0jmGR5K+B\nY+nNlnMpvdVCvwkcDVxSVX8zwHhDJcno6TIDHAn8M0BVPXfGQw2xJNcChzQzJi2hN4vS+cBRTfvz\nBxpwyCQ5HTgV+AlwKPDGqvpKs++KqnrSIPMNmyRnACcC59FbaRl661OcAJxXVWcOKtuwSfL9qjqs\nef86en9PLwCeCXzVezk1Se4ENgI/Bz4HfKGqbhlsqulloT1ASX5ZVfMHnWNYJLma3j+62wO/Avat\nqvVJdgD+taoOHmjAIZLkCuDHwCeAoldof47eP7xU1WWDSzd8kvykqh7fvH9AIZjkyqo6dHDphk/z\n3/ofV9Vvkiyg90PLp6vqg0l+WFV/NNCAQybJT4EnVNV9o9q3A66tqgMGk2z49P/9S7IcOK6qbkmy\nE3B5Vf3hYBMOlyQ/BP4T8AzgJcBzgR/Q+/foS1W1YYDxpsXQL8G+tUty1Xi7gL1mMssssKmq7gfu\nSvLzqloPUFV3J9k84GzDZiHwRuB/Am+pqiuT3G2BvcWu6fsN1Y+SLKyqFUkeB9w32cn6HdtU1W8A\nqmpVkqcD5yd5FL3/d2pqNgP7AL8Y1b53s0/tbZPkYfSG3s4Z6X2tqo1JNg022lCqqtoMfA34WpKH\n0PvN9YnA+4C5gww3HSy0u7cX8Czg9lHtAb4783GG2r1Jdqyqu+j9BAxAMwbWfyymoPkf298n+ULz\n9Wb8/8GD8Vrgg0neBtwKfC/JamB1s09Tc3OSQ6vqSoCmZ/vZwNmAPYZT9ybgG0l+Ru/vJMB84LHA\naQNLNZx2o9fjGqCS7F1VNzXPEvhD4NQ94J41v3VZCixNsuNgIk0vh450LMk/Ap+sqm+Pse/cqnrp\nAGINpSTbV9U9Y7TvAexdVVcPINaskOR44GlV9dZBZxlmzQOR+9P7oWVNVd084EhDKcm+9H6D9asx\n9j2tqr4zgFhDLck2wGHAvKZpLbC8+S2hHqSmKNyrqm4cdJZhkuRxVfXTQefokoW2JEmS1AGn95Mk\nSZI6YKEtSZIkdcBCW5IkSeqAhbYkSZLUAQttSZIkqQP/H0vZ9o3x/F4MAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x25a4506f5c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "modelfit(xgb1, X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.93154, std: 0.00250, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: 0.93152, std: 0.00256, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: 0.93156, std: 0.00251, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: 0.93098, std: 0.00271, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: 0.93113, std: 0.00274, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: 0.93117, std: 0.00270, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: 0.93033, std: 0.00293, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: 0.93047, std: 0.00290, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: 0.93081, std: 0.00283, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: 0.92994, std: 0.00309, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: 0.93018, std: 0.00293, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: 0.93059, std: 0.00284, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 3, 'min_child_weight': 5},\n",
       " 0.9315632351457767)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {\n",
    "    'max_depth': list(range(3,10,2)),\n",
    "    'min_child_weight':list(range(1,6,2))\n",
    "}\n",
    "gsearch1 = GridSearchCV(estimator=XGBClassifier(learning_rate =0.1, n_estimators=200, max_depth=5,\n",
    "min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
    " param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch1.fit(X,y)\n",
    "gsearch1.grid_scores_ ,gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.93122, std: 0.00272, params: {'max_depth': 2, 'min_child_weight': 4},\n",
       "  mean: 0.93122, std: 0.00267, params: {'max_depth': 2, 'min_child_weight': 5},\n",
       "  mean: 0.93122, std: 0.00266, params: {'max_depth': 2, 'min_child_weight': 6},\n",
       "  mean: 0.93152, std: 0.00258, params: {'max_depth': 3, 'min_child_weight': 4},\n",
       "  mean: 0.93156, std: 0.00251, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: 0.93153, std: 0.00261, params: {'max_depth': 3, 'min_child_weight': 6},\n",
       "  mean: 0.93155, std: 0.00265, params: {'max_depth': 4, 'min_child_weight': 4},\n",
       "  mean: 0.93149, std: 0.00260, params: {'max_depth': 4, 'min_child_weight': 5},\n",
       "  mean: 0.93151, std: 0.00264, params: {'max_depth': 4, 'min_child_weight': 6}],\n",
       " {'max_depth': 3, 'min_child_weight': 5},\n",
       " 0.9315632351457767)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {\n",
    "    'max_depth': list(range(2,5,1)),\n",
    "    'min_child_weight':list(range(4,7,1))\n",
    "}\n",
    "gsearch1 = GridSearchCV(estimator=XGBClassifier(learning_rate =0.1, n_estimators=200, max_depth=5,\n",
    "min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1, seed=27), \n",
    " param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch1.fit(X,y)\n",
    "gsearch1.grid_scores_ ,gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.93156, std: 0.00251, params: {'gamma': 0.0},\n",
       "  mean: 0.93155, std: 0.00254, params: {'gamma': 0.1},\n",
       "  mean: 0.93155, std: 0.00253, params: {'gamma': 0.2},\n",
       "  mean: 0.93150, std: 0.00258, params: {'gamma': 0.3},\n",
       "  mean: 0.93153, std: 0.00258, params: {'gamma': 0.4},\n",
       "  mean: 0.93160, std: 0.00261, params: {'gamma': 0.5},\n",
       "  mean: 0.93156, std: 0.00260, params: {'gamma': 0.6},\n",
       "  mean: 0.93150, std: 0.00266, params: {'gamma': 0.7},\n",
       "  mean: 0.93154, std: 0.00268, params: {'gamma': 0.8},\n",
       "  mean: 0.93159, std: 0.00265, params: {'gamma': 0.9}],\n",
       " {'gamma': 0.5},\n",
       " 0.9316014768521474)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test3 = {\n",
    " 'gamma':[i/10.0 for i in range(0,10)]\n",
    "}\n",
    "gsearch3 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=3,\n",
    " min_child_weight=5, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test3, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch3.fit(X,y)\n",
    "gsearch3.grid_scores_, gsearch3.best_params_, gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.93151, std: 0.00239, params: {'colsample_bytree': 0.4, 'subsample': 0.4},\n",
       "  mean: 0.93150, std: 0.00249, params: {'colsample_bytree': 0.4, 'subsample': 0.5},\n",
       "  mean: 0.93133, std: 0.00244, params: {'colsample_bytree': 0.4, 'subsample': 0.6},\n",
       "  mean: 0.93140, std: 0.00259, params: {'colsample_bytree': 0.4, 'subsample': 0.7},\n",
       "  mean: 0.93143, std: 0.00255, params: {'colsample_bytree': 0.4, 'subsample': 0.8},\n",
       "  mean: 0.93141, std: 0.00262, params: {'colsample_bytree': 0.4, 'subsample': 0.9},\n",
       "  mean: 0.93138, std: 0.00242, params: {'colsample_bytree': 0.5, 'subsample': 0.4},\n",
       "  mean: 0.93153, std: 0.00242, params: {'colsample_bytree': 0.5, 'subsample': 0.5},\n",
       "  mean: 0.93148, std: 0.00246, params: {'colsample_bytree': 0.5, 'subsample': 0.6},\n",
       "  mean: 0.93160, std: 0.00249, params: {'colsample_bytree': 0.5, 'subsample': 0.7},\n",
       "  mean: 0.93153, std: 0.00259, params: {'colsample_bytree': 0.5, 'subsample': 0.8},\n",
       "  mean: 0.93160, std: 0.00251, params: {'colsample_bytree': 0.5, 'subsample': 0.9},\n",
       "  mean: 0.93138, std: 0.00242, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: 0.93153, std: 0.00242, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: 0.93148, std: 0.00246, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: 0.93160, std: 0.00249, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: 0.93153, std: 0.00259, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: 0.93160, std: 0.00251, params: {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "  mean: 0.93139, std: 0.00249, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: 0.93155, std: 0.00242, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: 0.93146, std: 0.00249, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: 0.93148, std: 0.00264, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: 0.93158, std: 0.00252, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: 0.93153, std: 0.00246, params: {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  mean: 0.93164, std: 0.00266, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: 0.93157, std: 0.00255, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: 0.93154, std: 0.00255, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: 0.93148, std: 0.00263, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: 0.93155, std: 0.00264, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: 0.93148, std: 0.00243, params: {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  mean: 0.93156, std: 0.00258, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: 0.93148, std: 0.00251, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: 0.93148, std: 0.00258, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: 0.93165, std: 0.00264, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: 0.93158, std: 0.00263, params: {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       "  mean: 0.93156, std: 0.00234, params: {'colsample_bytree': 0.9, 'subsample': 0.9}],\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       " 0.9316516988674749)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test4 = {\n",
    " 'subsample':[i/10.0 for i in range(4,10)],\n",
    " 'colsample_bytree':[i/10.0 for i in range(4,10)]\n",
    "}\n",
    "gsearch4 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=3,\n",
    " min_child_weight=5, gamma=0.5, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test4, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch4.fit(X, y)\n",
    "gsearch4.grid_scores_, gsearch4.best_params_, gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.93155, std: 0.00266, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: 0.93139, std: 0.00250, params: {'colsample_bytree': 0.8, 'subsample': 0.65},\n",
       "  mean: 0.93158, std: 0.00250, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: 0.93169, std: 0.00258, params: {'colsample_bytree': 0.8, 'subsample': 0.75},\n",
       "  mean: 0.93155, std: 0.00266, params: {'colsample_bytree': 0.85, 'subsample': 0.6},\n",
       "  mean: 0.93139, std: 0.00250, params: {'colsample_bytree': 0.85, 'subsample': 0.65},\n",
       "  mean: 0.93158, std: 0.00250, params: {'colsample_bytree': 0.85, 'subsample': 0.7},\n",
       "  mean: 0.93169, std: 0.00258, params: {'colsample_bytree': 0.85, 'subsample': 0.75},\n",
       "  mean: 0.93150, std: 0.00253, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: 0.93129, std: 0.00259, params: {'colsample_bytree': 0.9, 'subsample': 0.65},\n",
       "  mean: 0.93172, std: 0.00258, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: 0.93157, std: 0.00260, params: {'colsample_bytree': 0.9, 'subsample': 0.75},\n",
       "  mean: 0.93150, std: 0.00253, params: {'colsample_bytree': 0.95, 'subsample': 0.6},\n",
       "  mean: 0.93129, std: 0.00259, params: {'colsample_bytree': 0.95, 'subsample': 0.65},\n",
       "  mean: 0.93172, std: 0.00258, params: {'colsample_bytree': 0.95, 'subsample': 0.7},\n",
       "  mean: 0.93157, std: 0.00260, params: {'colsample_bytree': 0.95, 'subsample': 0.75}],\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       " 0.9317213983691044)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test5 = {\n",
    " 'subsample':[i/100.0 for i in range(60,80,5)],\n",
    " 'colsample_bytree':[i/100.0 for i in range(80,100,5)]\n",
    "}\n",
    "gsearch5 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=3,\n",
    " min_child_weight=5, gamma=0.5, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test5, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch5.fit(X, y)\n",
    "gsearch5.grid_scores_, gsearch5.best_params_, gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([mean: 0.93161, std: 0.00268, params: {'reg_alpha': 1e-05},\n",
       "  mean: 0.93166, std: 0.00263, params: {'reg_alpha': 0.01},\n",
       "  mean: 0.93161, std: 0.00268, params: {'reg_alpha': 0},\n",
       "  mean: 0.93164, std: 0.00270, params: {'reg_alpha': 0.1},\n",
       "  mean: 0.93164, std: 0.00272, params: {'reg_alpha': 1},\n",
       "  mean: 0.92725, std: 0.00268, params: {'reg_alpha': 100}],\n",
       " {'reg_alpha': 0.01},\n",
       " 0.9316555272063324)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test6 = {\n",
    " 'reg_alpha':[1e-5, 1e-2, 0, 0.1, 1, 100]\n",
    "}\n",
    "gsearch6 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=200, max_depth=3,\n",
    " min_child_weight=5, gamma=0.5, subsample=0.7, colsample_bytree=0.9,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test6, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch6.fit(X, y)\n",
    "gsearch6.grid_scores_, gsearch6.best_params_, gsearch6.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}