playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import gzip\n",
    "import seaborn as sns\n",
    "from pandas.io.json import json_normalize\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open Catalog dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "catalog = pd.read_csv('data/catalog.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>current_price</th>\n",
       "      <th>original_price</th>\n",
       "      <th>category</th>\n",
       "      <th>sub_category</th>\n",
       "      <th>sub_sub_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c9fe04e5097c087b6b6eeccc3adc4a142e14aa46</td>\n",
       "      <td>699.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>c1bd5fd999bd577743936898ada96496b547af3c</td>\n",
       "      <td>f08770a96fb546673053ab799f5ea9cada06c06a</td>\n",
       "      <td>2d2c44a2d8f18a6271f0e8057313af68a46d0f24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e00988f42f7fc5f14ec6a0e7905789110f23c5de</td>\n",
       "      <td>150.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37b2661cc181c987f68770e43990152026920ba5</td>\n",
       "      <td>11e623a37e87cf7995c466723ec99688d55cae8c</td>\n",
       "      <td>11e623a37e87cf7995c466723ec99688d55cae8c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1e2e7c0d4082295728c1684a702cb43e1d332d3e</td>\n",
       "      <td>549.9</td>\n",
       "      <td>1049.9</td>\n",
       "      <td>c37df0ae71b97699a478def3001a3516a905a51d</td>\n",
       "      <td>f08770a96fb546673053ab799f5ea9cada06c06a</td>\n",
       "      <td>d6605426aaa703bf19874c0caac79a661b73de33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a279dd2284eb57533ca417c258ede0a0526a6f6e</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>c1bd5fd999bd577743936898ada96496b547af3c</td>\n",
       "      <td>6d9d48ae11ee1909235f31e9bfe5d36aa1462cb3</td>\n",
       "      <td>9a2e3cb56f6a1756fd35a4fd70172a67ecf13639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>405b0362ae6ec149700164811b0e7773c8300e9d</td>\n",
       "      <td>79.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>c37df0ae71b97699a478def3001a3516a905a51d</td>\n",
       "      <td>78af0aac89e0f15a6c9fc70b5bff79d98c6dcc43</td>\n",
       "      <td>2541a58f702844477aab540e5df7b859a1e3d5de</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        pid  current_price  original_price  \\\n",
       "0  c9fe04e5097c087b6b6eeccc3adc4a142e14aa46          699.0             NaN   \n",
       "1  e00988f42f7fc5f14ec6a0e7905789110f23c5de          150.0             NaN   \n",
       "2  1e2e7c0d4082295728c1684a702cb43e1d332d3e          549.9          1049.9   \n",
       "3  a279dd2284eb57533ca417c258ede0a0526a6f6e            0.0             0.0   \n",
       "4  405b0362ae6ec149700164811b0e7773c8300e9d           79.9             0.0   \n",
       "\n",
       "                                   category  \\\n",
       "0  c1bd5fd999bd577743936898ada96496b547af3c   \n",
       "1  37b2661cc181c987f68770e43990152026920ba5   \n",
       "2  c37df0ae71b97699a478def3001a3516a905a51d   \n",
       "3  c1bd5fd999bd577743936898ada96496b547af3c   \n",
       "4  c37df0ae71b97699a478def3001a3516a905a51d   \n",
       "\n",
       "                               sub_category  \\\n",
       "0  f08770a96fb546673053ab799f5ea9cada06c06a   \n",
       "1  11e623a37e87cf7995c466723ec99688d55cae8c   \n",
       "2  f08770a96fb546673053ab799f5ea9cada06c06a   \n",
       "3  6d9d48ae11ee1909235f31e9bfe5d36aa1462cb3   \n",
       "4  78af0aac89e0f15a6c9fc70b5bff79d98c6dcc43   \n",
       "\n",
       "                           sub_sub_category  \n",
       "0  2d2c44a2d8f18a6271f0e8057313af68a46d0f24  \n",
       "1  11e623a37e87cf7995c466723ec99688d55cae8c  \n",
       "2  d6605426aaa703bf19874c0caac79a661b73de33  \n",
       "3  9a2e3cb56f6a1756fd35a4fd70172a67ecf13639  \n",
       "4  2541a58f702844477aab540e5df7b859a1e3d5de  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "catalog.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open dataset with a JSON object per line with NaN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 ms, sys: 0 ns, total: 4 ms\n",
      "Wall time: 2.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with gzip.open('data/test.gz', 'r') as f:\n",
    "    json_records = []\n",
    "    for line in f:\n",
    "        record = json.loads(line)\n",
    "        json_records.append(record)\n",
    "data = pd.DataFrame(json_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>event_type</th>\n",
       "      <th>page_type</th>\n",
       "      <th>productId</th>\n",
       "      <th>products</th>\n",
       "      <th>source</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>pageview</td>\n",
       "      <td>cart</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-01</td>\n",
       "      <td>54cfe1703c536e98bf02c7c4cd5b4e280f07d74c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>pageview</td>\n",
       "      <td>checkout</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>c808d5ecb73bee00a9730064b7652bec10778f20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>pageview</td>\n",
       "      <td>checkout</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>7ba0cbaaf2000d440b474ff9182383c5525d0d56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>pageview</td>\n",
       "      <td>confirmation</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>3b2790a1f746e59298c8c277e3ec2279343ce52d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>pageview</td>\n",
       "      <td>checkout</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-01-02</td>\n",
       "      <td>3b2790a1f746e59298c8c277e3ec2279343ce52d</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  date event_type     page_type productId products source   timestamp  \\\n",
       "0  NaN   pageview          cart       NaN      NaN    NaN  2016-01-01   \n",
       "1  NaN   pageview      checkout       NaN      NaN    NaN  2016-01-02   \n",
       "2  NaN   pageview      checkout       NaN      NaN    NaN  2016-01-02   \n",
       "3  NaN   pageview  confirmation       NaN      NaN    NaN  2016-01-02   \n",
       "4  NaN   pageview      checkout       NaN      NaN    NaN  2016-01-02   \n",
       "\n",
       "                                        uid  \n",
       "0  54cfe1703c536e98bf02c7c4cd5b4e280f07d74c  \n",
       "1  c808d5ecb73bee00a9730064b7652bec10778f20  \n",
       "2  7ba0cbaaf2000d440b474ff9182383c5525d0d56  \n",
       "3  3b2790a1f746e59298c8c277e3ec2279343ce52d  \n",
       "4  3b2790a1f746e59298c8c277e3ec2279343ce52d  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Open only purchase events from data dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 532 ms, sys: 60 ms, total: 592 ms\n",
      "Wall time: 666 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open('data/purchase_data', 'r') as f:\n",
    "    json_records = []\n",
    "    for line in f:\n",
    "        record = json.loads(line)\n",
    "        json_records.append(record)\n",
    "purchase = pd.DataFrame(json_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>event_type</th>\n",
       "      <th>gender</th>\n",
       "      <th>products</th>\n",
       "      <th>source</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>purchase</td>\n",
       "      <td>M</td>\n",
       "      <td>[{u'pid': u'db3eae1855619bd4462848bb7b473b2193...</td>\n",
       "      <td>desktop</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2016-05-26 11:21:40</td>\n",
       "      <td>purchase</td>\n",
       "      <td>M</td>\n",
       "      <td>[{u'pid': u'418e19155782fc7f12ced8332c8ed025ae...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>07be0ebd1e6f2412e4a9a3f679ded05796a1e225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>purchase</td>\n",
       "      <td>M</td>\n",
       "      <td>[{u'pid': u'4b17c3aa28d53208c9fd147057b40fdd81...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-11-07 11:36:00</td>\n",
       "      <td>purchase</td>\n",
       "      <td>M</td>\n",
       "      <td>[{u'pid': u'acd9858f0d4d4c61ee783080a1a305b4f0...</td>\n",
       "      <td>desktop</td>\n",
       "      <td>7f1c45f10de66a7023bfa9f9ed053c5b3ae03f56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-08-28 20:48:58</td>\n",
       "      <td>purchase</td>\n",
       "      <td>M</td>\n",
       "      <td>[{u'pid': u'fca1dc5059edb8a8ee2fa424ce7a01958b...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>79ddd2b0c65947a8a11249c2fa71499055507e11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  date event_type gender  \\\n",
       "0  2017-03-27 14:19:00   purchase      M   \n",
       "1  2016-05-26 11:21:40   purchase      M   \n",
       "2  2016-03-05 06:52:27   purchase      M   \n",
       "3  2016-11-07 11:36:00   purchase      M   \n",
       "4  2016-08-28 20:48:58   purchase      M   \n",
       "\n",
       "                                            products   source  \\\n",
       "0  [{u'pid': u'db3eae1855619bd4462848bb7b473b2193...  desktop   \n",
       "1  [{u'pid': u'418e19155782fc7f12ced8332c8ed025ae...      NaN   \n",
       "2  [{u'pid': u'4b17c3aa28d53208c9fd147057b40fdd81...      NaN   \n",
       "3  [{u'pid': u'acd9858f0d4d4c61ee783080a1a305b4f0...  desktop   \n",
       "4  [{u'pid': u'fca1dc5059edb8a8ee2fa424ce7a01958b...      NaN   \n",
       "\n",
       "                                        uid  \n",
       "0  1f9acd4f729c04e18e5d72bebca5e5524e67687e  \n",
       "1  07be0ebd1e6f2412e4a9a3f679ded05796a1e225  \n",
       "2  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b  \n",
       "3  7f1c45f10de66a7023bfa9f9ed053c5b3ae03f56  \n",
       "4  79ddd2b0c65947a8a11249c2fa71499055507e11  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "purchase.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unnest products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 580 ms, sys: 16 ms, total: 596 ms\n",
      "Wall time: 595 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open('data/purchase_data', 'r') as f:\n",
    "    json_records = []\n",
    "    for line in f:\n",
    "        record = json.loads(line)\n",
    "        products = record.pop('products')\n",
    "        record.pop('source')\n",
    "        record.pop('event_type')\n",
    "        for obj in products:\n",
    "            new_record = record.copy()\n",
    "            new_record.update(obj)\n",
    "            json_records.append(new_record)\n",
    "purchase = pd.DataFrame(json_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>gender</th>\n",
       "      <th>pid</th>\n",
       "      <th>quantity</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>M</td>\n",
       "      <td>db3eae1855619bd4462848bb7b473b219345ee87</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>M</td>\n",
       "      <td>3aed9b0313f9226111de8aeabaedccf8db07d428</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2016-05-26 11:21:40</td>\n",
       "      <td>M</td>\n",
       "      <td>418e19155782fc7f12ced8332c8ed025aec227a5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>07be0ebd1e6f2412e4a9a3f679ded05796a1e225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>M</td>\n",
       "      <td>4b17c3aa28d53208c9fd147057b40fdd8108f790</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>M</td>\n",
       "      <td>31771f432cbc8e7ee3140f70b1e866fb5a2d739a</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  date gender                                       pid  \\\n",
       "0  2017-03-27 14:19:00      M  db3eae1855619bd4462848bb7b473b219345ee87   \n",
       "1  2017-03-27 14:19:00      M  3aed9b0313f9226111de8aeabaedccf8db07d428   \n",
       "2  2016-05-26 11:21:40      M  418e19155782fc7f12ced8332c8ed025aec227a5   \n",
       "3  2016-03-05 06:52:27      M  4b17c3aa28d53208c9fd147057b40fdd8108f790   \n",
       "4  2016-03-05 06:52:27      M  31771f432cbc8e7ee3140f70b1e866fb5a2d739a   \n",
       "\n",
       "   quantity                                       uid  \n",
       "0       1.0  1f9acd4f729c04e18e5d72bebca5e5524e67687e  \n",
       "1       1.0  1f9acd4f729c04e18e5d72bebca5e5524e67687e  \n",
       "2       1.0  07be0ebd1e6f2412e4a9a3f679ded05796a1e225  \n",
       "3       1.0  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b  \n",
       "4       1.0  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "purchase.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using json_normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 776 ms, sys: 24 ms, total: 800 ms\n",
      "Wall time: 800 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open('data/purchase_data', 'r') as f:\n",
    "    json_records = []\n",
    "    for line in f:\n",
    "        record = json.loads(line)\n",
    "        json_records.append(record)\n",
    "purchase = json_normalize(json_records, 'products', ['uid','date','gender'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>quantity</th>\n",
       "      <th>date</th>\n",
       "      <th>gender</th>\n",
       "      <th>uid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>db3eae1855619bd4462848bb7b473b219345ee87</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>M</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3aed9b0313f9226111de8aeabaedccf8db07d428</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>M</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>418e19155782fc7f12ced8332c8ed025aec227a5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-05-26 11:21:40</td>\n",
       "      <td>M</td>\n",
       "      <td>07be0ebd1e6f2412e4a9a3f679ded05796a1e225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4b17c3aa28d53208c9fd147057b40fdd8108f790</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>M</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31771f432cbc8e7ee3140f70b1e866fb5a2d739a</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>M</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        pid  quantity                 date  \\\n",
       "0  db3eae1855619bd4462848bb7b473b219345ee87       1.0  2017-03-27 14:19:00   \n",
       "1  3aed9b0313f9226111de8aeabaedccf8db07d428       1.0  2017-03-27 14:19:00   \n",
       "2  418e19155782fc7f12ced8332c8ed025aec227a5       1.0  2016-05-26 11:21:40   \n",
       "3  4b17c3aa28d53208c9fd147057b40fdd8108f790       1.0  2016-03-05 06:52:27   \n",
       "4  31771f432cbc8e7ee3140f70b1e866fb5a2d739a       1.0  2016-03-05 06:52:27   \n",
       "\n",
       "  gender                                       uid  \n",
       "0      M  1f9acd4f729c04e18e5d72bebca5e5524e67687e  \n",
       "1      M  1f9acd4f729c04e18e5d72bebca5e5524e67687e  \n",
       "2      M  07be0ebd1e6f2412e4a9a3f679ded05796a1e225  \n",
       "3      M  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b  \n",
       "4      M  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "purchase.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "purchase = purchase.join(catalog.set_index('pid'), on='pid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 45628 entries, 0 to 45627\n",
      "Data columns (total 10 columns):\n",
      "pid                 45628 non-null object\n",
      "quantity            45628 non-null float64\n",
      "date                45628 non-null object\n",
      "gender              45628 non-null object\n",
      "uid                 45628 non-null object\n",
      "current_price       45622 non-null float64\n",
      "original_price      36256 non-null float64\n",
      "category            45628 non-null object\n",
      "sub_category        45628 non-null object\n",
      "sub_sub_category    45628 non-null object\n",
      "dtypes: float64(3), object(7)\n",
      "memory usage: 3.5+ MB\n"
     ]
    }
   ],
   "source": [
    "purchase.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 45628 entries, 0 to 45627\n",
      "Columns: 109 entries, pid to gender_M\n",
      "dtypes: float64(3), object(3), uint8(103)\n",
      "memory usage: 6.6+ MB\n"
     ]
    }
   ],
   "source": [
    "categorical = ['category', 'sub_category', 'sub_sub_category','gender']\n",
    "new_purchase = pd.get_dummies(purchase, columns=categorical)\n",
    "new_purchase.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pid</th>\n",
       "      <th>quantity</th>\n",
       "      <th>date</th>\n",
       "      <th>uid</th>\n",
       "      <th>current_price</th>\n",
       "      <th>original_price</th>\n",
       "      <th>category_11e623a37e87cf7995c466723ec99688d55cae8c</th>\n",
       "      <th>category_37b2661cc181c987f68770e43990152026920ba5</th>\n",
       "      <th>category_55599ee4735544f9319a030222f617a252d8ebf3</th>\n",
       "      <th>category_57ecb679698676e7a3ac843bef1e3f6844651a79</th>\n",
       "      <th>...</th>\n",
       "      <th>sub_sub_category_eb0619cfb60da489f61001b19331dad5d480f82b</th>\n",
       "      <th>sub_sub_category_ec6c67dfdd9a89bb590c67bdc7865bfdd2a810ee</th>\n",
       "      <th>sub_sub_category_ef76dd0072150112992c6db54e2a7d3d11c302cb</th>\n",
       "      <th>sub_sub_category_f08770a96fb546673053ab799f5ea9cada06c06a</th>\n",
       "      <th>sub_sub_category_f85ea193363a91d5d03995129ec35786cfa77e6f</th>\n",
       "      <th>sub_sub_category_f86a9cce8e46d9cac88fd9306a6515fef89fe3fc</th>\n",
       "      <th>sub_sub_category_f9e79addaa1175027c52d95685b4664fa7430f0f</th>\n",
       "      <th>sub_sub_category_fecdd01f32c4b0a1e0d2b70cc3891dc52f964329</th>\n",
       "      <th>gender_F</th>\n",
       "      <th>gender_M</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>db3eae1855619bd4462848bb7b473b219345ee87</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "      <td>529.9</td>\n",
       "      <td>899.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3aed9b0313f9226111de8aeabaedccf8db07d428</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2017-03-27 14:19:00</td>\n",
       "      <td>1f9acd4f729c04e18e5d72bebca5e5524e67687e</td>\n",
       "      <td>469.9</td>\n",
       "      <td>799.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>418e19155782fc7f12ced8332c8ed025aec227a5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-05-26 11:21:40</td>\n",
       "      <td>07be0ebd1e6f2412e4a9a3f679ded05796a1e225</td>\n",
       "      <td>249.9</td>\n",
       "      <td>429.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4b17c3aa28d53208c9fd147057b40fdd8108f790</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "      <td>640.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31771f432cbc8e7ee3140f70b1e866fb5a2d739a</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016-03-05 06:52:27</td>\n",
       "      <td>3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b</td>\n",
       "      <td>129.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 109 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        pid  quantity                 date  \\\n",
       "0  db3eae1855619bd4462848bb7b473b219345ee87       1.0  2017-03-27 14:19:00   \n",
       "1  3aed9b0313f9226111de8aeabaedccf8db07d428       1.0  2017-03-27 14:19:00   \n",
       "2  418e19155782fc7f12ced8332c8ed025aec227a5       1.0  2016-05-26 11:21:40   \n",
       "3  4b17c3aa28d53208c9fd147057b40fdd8108f790       1.0  2016-03-05 06:52:27   \n",
       "4  31771f432cbc8e7ee3140f70b1e866fb5a2d739a       1.0  2016-03-05 06:52:27   \n",
       "\n",
       "                                        uid  current_price  original_price  \\\n",
       "0  1f9acd4f729c04e18e5d72bebca5e5524e67687e          529.9           899.9   \n",
       "1  1f9acd4f729c04e18e5d72bebca5e5524e67687e          469.9           799.9   \n",
       "2  07be0ebd1e6f2412e4a9a3f679ded05796a1e225          249.9           429.0   \n",
       "3  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b          640.0             NaN   \n",
       "4  3cf9949b8f90daadd8f67ec46214e64d6b9cfa5b          129.9             0.0   \n",
       "\n",
       "   category_11e623a37e87cf7995c466723ec99688d55cae8c  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   category_37b2661cc181c987f68770e43990152026920ba5  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   category_55599ee4735544f9319a030222f617a252d8ebf3  \\\n",
       "0                                                  0   \n",
       "1                                                  0   \n",
       "2                                                  0   \n",
       "3                                                  0   \n",
       "4                                                  0   \n",
       "\n",
       "   category_57ecb679698676e7a3ac843bef1e3f6844651a79    ...     \\\n",
       "0                                                  0    ...      \n",
       "1                                                  0    ...      \n",
       "2                                                  0    ...      \n",
       "3                                                  0    ...      \n",
       "4                                                  0    ...      \n",
       "\n",
       "   sub_sub_category_eb0619cfb60da489f61001b19331dad5d480f82b  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  1           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_ec6c67dfdd9a89bb590c67bdc7865bfdd2a810ee  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_ef76dd0072150112992c6db54e2a7d3d11c302cb  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_f08770a96fb546673053ab799f5ea9cada06c06a  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_f85ea193363a91d5d03995129ec35786cfa77e6f  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_f86a9cce8e46d9cac88fd9306a6515fef89fe3fc  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_f9e79addaa1175027c52d95685b4664fa7430f0f  \\\n",
       "0                                                  0           \n",
       "1                                                  0           \n",
       "2                                                  0           \n",
       "3                                                  0           \n",
       "4                                                  0           \n",
       "\n",
       "   sub_sub_category_fecdd01f32c4b0a1e0d2b70cc3891dc52f964329  gender_F  \\\n",
       "0                                                  0                 0   \n",
       "1                                                  0                 0   \n",
       "2                                                  0                 0   \n",
       "3                                                  0                 0   \n",
       "4                                                  0                 0   \n",
       "\n",
       "   gender_M  \n",
       "0         1  \n",
       "1         1  \n",
       "2         1  \n",
       "3         1  \n",
       "4         1  \n",
       "\n",
       "[5 rows x 109 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_purchase.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 45628 entries, 0 to 45627\n",
      "Columns: 105 entries, pid to gender_M\n",
      "dtypes: float64(3), object(3), uint8(99)\n",
      "memory usage: 6.4+ MB\n"
     ]
    }
   ],
   "source": [
    "dummy_cols = ['category_c1bd5fd999bd577743936898ada96496b547af3c',\n",
    "'sub_category_f08770a96fb546673053ab799f5ea9cada06c06a',\n",
    "'sub_sub_category_2d2c44a2d8f18a6271f0e8057313af68a46d0f24',\n",
    "'gender_F']\n",
    "purchase2 = new_purchase.drop(dummy_cols, 1)\n",
    "purchase2.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Machine Learning Algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn import linear_model\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.calibration import calibration_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = purchase2.columns[6:-1]\n",
    "target = purchase2.columns[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gambiarra : nem todas as features tão no target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 256 ms, sys: 0 ns, total: 256 ms\n",
      "Wall time: 256 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open('data/purchase_target', 'r') as f:\n",
    "    json_records = []\n",
    "    for line in f:\n",
    "        record = json.loads(line)\n",
    "        json_records.append(record)\n",
    "purchase3 = json_normalize(json_records, 'products', ['uid','date'])\n",
    "purchase3 = purchase3.join(catalog.set_index('pid'), on='pid')\n",
    "categorical = ['category', 'sub_category', 'sub_sub_category']\n",
    "new_purchase2 = pd.get_dummies(purchase3, columns=categorical)\n",
    "dummy_cols = ['category_c1bd5fd999bd577743936898ada96496b547af3c',\n",
    "'sub_category_f08770a96fb546673053ab799f5ea9cada06c06a',\n",
    "'sub_sub_category_2d2c44a2d8f18a6271f0e8057313af68a46d0f24']\n",
    "purchase4 = new_purchase2.drop(dummy_cols, 1)\n",
    "features2 = purchase4.columns[6:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_features = list(set(features)&set(features2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = purchase2[new_features]\n",
    "Y = purchase2[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.4, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x_train.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 76 ms, sys: 12 ms, total: 88 ms\n",
      "Wall time: 86.4 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "gnb = GaussianNB()\n",
    "gnb.fit(x_train, y_train)\n",
    "y_pred = gnb.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.48197457812842426"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 10s, sys: 60 ms, total: 1min 10s\n",
      "Wall time: 1min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "knn = KNeighborsClassifier(n_neighbors=1)\n",
    "knn.fit(x_train, y_train)\n",
    "y_pred = knn.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.71137409598948065"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 176 ms, sys: 0 ns, total: 176 ms\n",
      "Wall time: 172 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tree = DecisionTreeClassifier()\n",
    "tree.fit(x_train, y_train)\n",
    "y_pred = tree.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75345167652859957"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forest Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.37 s, sys: 0 ns, total: 6.37 s\n",
      "Wall time: 6.36 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "forest = RandomForestClassifier(n_estimators=100)\n",
    "forest.fit(x_train, y_train)\n",
    "y_pred = tree.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75345167652859957"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.45 s, sys: 4 ms, total: 5.46 s\n",
      "Wall time: 5.45 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "boost = GradientBoostingClassifier()\n",
    "boost.fit(x_train, y_train)\n",
    "y_pred = boost.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75076703922857768"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = linear_model.SGDClassifier()\n",
    "clf.fit(x_train, y_train)\n",
    "y_pred = clf.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.74786324786324787"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 452 ms, sys: 8 ms, total: 460 ms\n",
      "Wall time: 452 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "clf = linear_model.LogisticRegression(C=1e5)\n",
    "clf.fit(x_train, y_train)\n",
    "y_pred = clf.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.74983563445101908"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 356 ms, sys: 0 ns, total: 356 ms\n",
      "Wall time: 345 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "svc = LinearSVC(C=1.0)\n",
    "svc.fit(x_train, y_train)\n",
    "y_pred = svc.predict(x_test)\n",
    "score = metrics.accuracy_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.74983563445101908"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "# Generate output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = purchase4[new_features]\n",
    "answer = tree.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12681"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12681"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users = purchase4.uid.values\n",
    "len(users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3209"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(users))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ans = []\n",
    "for i,u in enumerate(users):\n",
    "    if answer[i]:\n",
    "        g = 'M'\n",
    "    else:\n",
    "        g = 'F'\n",
    "    obj = {'a':u, 'b':g}\n",
    "    ans.append(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "with open('ans.csv', 'wb') as f:\n",
    "    w = csv.DictWriter(f, fieldnames=['a','b'])\n",
    "    for obj in ans:\n",
    "        w.writerow(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('ans79.csv', 'r') as f:\n",
    "    w = csv.DictWriter(f, fieldnames=['a','b'])\n",
    "    for obj in ans:\n",
    "        w.writerow(obj)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "it = 100\n",
    "targ_X = target_data[features]\n",
    "hack_answer = clf.predict(targ_X)\n",
    "score = metrics.accuracy_score(y_test, y_pred)\n",
    "for i in range(it-1):\n",
    "    x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.33, random_state=i)\n",
    "    clf.fit(x_train, y_train)\n",
    "    y_pred = clf.predict(x_test)\n",
    "    score1 = metrics.accuracy_score(y_test, y_pred)\n",
    "    score = score + score1\n",
    "    new = clf.predict(targ_X)\n",
    "    hack_answer = hack_answer + new\n",
    "    \n",
    "    forest.fit(x_train, y_train)\n",
    "    y_pred = forest.predict(x_test)\n",
    "    score1 = metrics.accuracy_score(y_test, y_pred)\n",
    "    score = score + score1\n",
    "    new = clf.predict(targ_X)\n",
    "    hack_answer = hack_answer + new\n",
    "hack_answer = np.round((1.0*hack_answer)/it)\n",
    "score = (1.0*score)/it\n",
    "\n",
    "users = target_data.uid.values\n",
    "ans = []\n",
    "for i,u in enumerate(users):\n",
    "    if hack_answer[i]:\n",
    "        g = 'M'\n",
    "    else:\n",
    "        g = 'F'\n",
    "    obj = {'a':u, 'b':g}\n",
    "    ans.append(obj)\n",
    "    \n",
    "import csv\n",
    "\n",
    "with open('hack1.csv', 'wb') as f:\n",
    "    w = csv.DictWriter(f, fieldnames=['a','b'])\n",
    "    for obj in ans:\n",
    "        w.writerow(obj)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}