Read_CDN_Features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Item (CDN) Features Preprocessing\n",
    "\n",
    "**Input:** Raw data tables/logs (hidden for future research purposes)\n",
    "\n",
    "**Output:** Array of CDN feature vectors in one-hot format\n",
    "\n",
    "**Features used:**\n",
    "* Unique CDN code\n",
    "* CDN type (free/self-made/commercial)\n",
    "* Number of CDN IP addresses (binned by quartile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.sparse as sp\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "interactions_pickle = '20170629-interactions-mappings.pkl'\n",
    "with open(interactions_pickle, 'rb') as output:\n",
    "    (interactions, iidx_to_cdn, cdn_to_iidx, uidx_to_icp, icp_to_uidx) = pickle.load(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read in cdn_ips file\n",
    "cdn_ips_filepath = 'CDN_ips/all_cdn_ips.txt'\n",
    "cdn_ips_header = ['cdn', 'location', 'isp', 'ips', 'ts']\n",
    "cdn_ips_datatypes = {\n",
    "    'cdn': str,\n",
    "    'location': str,\n",
    "    'isp': str,\n",
    "    'ips': np.int64,\n",
    "    'ts': str\n",
    "}\n",
    "\n",
    "cdn_ips_df = pd.read_csv(cdn_ips_filepath, \n",
    "                              sep=',', header=None, \n",
    "                              names=cdn_ips_header,\n",
    "                              dtype=cdn_ips_datatypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cdn</th>\n",
       "      <th>location</th>\n",
       "      <th>isp</th>\n",
       "      <th>ips</th>\n",
       "      <th>ts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001</td>\n",
       "      <td>1101</td>\n",
       "      <td>10</td>\n",
       "      <td>99</td>\n",
       "      <td>201501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001</td>\n",
       "      <td>1101</td>\n",
       "      <td>11</td>\n",
       "      <td>24</td>\n",
       "      <td>201501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001</td>\n",
       "      <td>1101</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>201501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001</td>\n",
       "      <td>1200</td>\n",
       "      <td>10</td>\n",
       "      <td>19</td>\n",
       "      <td>201501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001</td>\n",
       "      <td>1200</td>\n",
       "      <td>11</td>\n",
       "      <td>122</td>\n",
       "      <td>201501</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cdn location isp  ips      ts\n",
       "0  001     1101  10   99  201501\n",
       "1  001     1101  11   24  201501\n",
       "2  001     1101  12    5  201501\n",
       "3  001     1200  10   19  201501\n",
       "4  001     1200  11  122  201501"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_ips_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read in cdn_ip_locations file\n",
    "cdn_ip_locations_filepath = 'CDN_IP_Locations/all_cdn_ip_locations.txt'\n",
    "cdn_ip_locations_header = ['cdn', 'ip', 'location', 'isp', 'ts']\n",
    "cdn_ip_locations_dtypes = {\n",
    "    'cdn': str,\n",
    "    'ip': str,\n",
    "    'location': str,\n",
    "    'isp': str,\n",
    "    'ts': str\n",
    "}\n",
    "\n",
    "cdn_ip_locations_df = pd.read_csv(cdn_ip_locations_filepath, \n",
    "                              sep=',', header=None, \n",
    "                              names=cdn_ip_locations_header,\n",
    "                              dtype=cdn_ip_locations_dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cdn\n",
       "001    9857\n",
       "002    3223\n",
       "003    1949\n",
       "004     372\n",
       "005     499\n",
       "006      83\n",
       "007      15\n",
       "008       1\n",
       "009      17\n",
       "011       1\n",
       "013     140\n",
       "201     231\n",
       "202     306\n",
       "203     207\n",
       "204     363\n",
       "Name: ip, dtype: int64"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get no. unique IPs for each CDN\n",
    "cdn_num_ips = cdn_ip_locations_df.groupby('cdn')['ip'].nunique()\n",
    "cdn_num_ips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# No. CDNs represented in each file\n",
    "print cdn_ips_df['cdn'].nunique()\n",
    "print cdn_ip_locations_df['cdn'].nunique()\n",
    "# https://stackoverflow.com/questions/38309729/count-unique-values-with-pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create CDN feature dict with CDN type (encoded in 1st digit of CDN code)\n",
    "# 0: free\n",
    "# 1: self-built\n",
    "# 2: commercial\n",
    "cdn_feature_dict = { cdn:dict(type=cdn[0]) for cdn in cdn_to_iidx }\n",
    "len(cdn_feature_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': '2'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_feature_dict['201']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cdn_temp_dns_filepath = '~/Desktop/CDN Data/cdn_temp_dns/000009_0'\n",
    "cdn_temp_dns_header = ['cdn', 'isp', 'cname', 'location', 'delay', 'time', 'ip', 'ts']\n",
    "cdn_temp_dns_dtypes = {\n",
    "    'cdn': str,\n",
    "    'isp': str,\n",
    "    'cname': str,\n",
    "    'location': str,\n",
    "    'delay': str,\n",
    "    'time': str,\n",
    "    'ip': str,\n",
    "    'ts': str\n",
    "}\n",
    "\n",
    "cdn_temp_dns = pd.read_csv(cdn_temp_dns_filepath, \n",
    "                              sep=',', header=None, \n",
    "                              names=cdn_temp_dns_header,\n",
    "                              dtype=cdn_temp_dns_dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cdn</th>\n",
       "      <th>isp</th>\n",
       "      <th>cname</th>\n",
       "      <th>location</th>\n",
       "      <th>delay</th>\n",
       "      <th>time</th>\n",
       "      <th>ip</th>\n",
       "      <th>ts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>002</td>\n",
       "      <td>11</td>\n",
       "      <td>v.pcgames.com.cn.wscdns.com</td>\n",
       "      <td>1201</td>\n",
       "      <td>3891</td>\n",
       "      <td>20150329005953</td>\n",
       "      <td>219.136.245.206</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004</td>\n",
       "      <td>11</td>\n",
       "      <td>pic9.huitu.com.cloudcdn.net</td>\n",
       "      <td>1201</td>\n",
       "      <td>28923</td>\n",
       "      <td>20150329005953</td>\n",
       "      <td>60.191.223.86</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001</td>\n",
       "      <td>11</td>\n",
       "      <td>user.shuuemura.ccgslb.net</td>\n",
       "      <td>1201</td>\n",
       "      <td>12644</td>\n",
       "      <td>20150329005953</td>\n",
       "      <td>220.181.66.132</td>\n",
       "      <td>220.181.46.163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>201</td>\n",
       "      <td>11</td>\n",
       "      <td>user.ourhost.com.cn.aqb.so</td>\n",
       "      <td>1201</td>\n",
       "      <td>57362</td>\n",
       "      <td>20150329005953</td>\n",
       "      <td>220.181.135.166</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001</td>\n",
       "      <td>11</td>\n",
       "      <td>user.kiehls.ccgslb.net</td>\n",
       "      <td>1201</td>\n",
       "      <td>9526</td>\n",
       "      <td>20150329005953</td>\n",
       "      <td>220.181.46.163</td>\n",
       "      <td>220.181.66.132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cdn isp                        cname location  delay            time  \\\n",
       "0  002  11  v.pcgames.com.cn.wscdns.com     1201   3891  20150329005953   \n",
       "1  004  11  pic9.huitu.com.cloudcdn.net     1201  28923  20150329005953   \n",
       "2  001  11    user.shuuemura.ccgslb.net     1201  12644  20150329005953   \n",
       "3  201  11   user.ourhost.com.cn.aqb.so     1201  57362  20150329005953   \n",
       "4  001  11       user.kiehls.ccgslb.net     1201   9526  20150329005953   \n",
       "\n",
       "                ip              ts  \n",
       "0  219.136.245.206          201503  \n",
       "1    60.191.223.86          201503  \n",
       "2   220.181.66.132  220.181.46.163  \n",
       "3  220.181.135.166          201503  \n",
       "4   220.181.46.163  220.181.66.132  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_temp_dns.head()\n",
    "# Looked at .csv file\n",
    "# Looks like there can be multiple 'ip's listed before 'ts'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lucashu/anaconda/envs/recsys/lib/python2.7/site-packages/pandas/core/groupby.py:3081: FutureWarning: numpy not_equal will not check object identity in the future. The comparison did not return the same result as suggested by the identity (`is`)) and will change.\n",
      "  inc = np.r_[1, val[1:] != val[:-1]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "cdn\n",
       "001    4829\n",
       "002    3262\n",
       "003    1760\n",
       "004     380\n",
       "005     482\n",
       "006      92\n",
       "007      18\n",
       "008       2\n",
       "009      20\n",
       "011       2\n",
       "013     120\n",
       "201     236\n",
       "202     308\n",
       "203     209\n",
       "204     359\n",
       "Name: ip, dtype: int64"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Appears to be same CDN list as in CDN_ip_locations!\n",
    "# Tested for all cdn_temp_dns files --> confirmed\n",
    "print cdn_temp_dns['cdn'].nunique()\n",
    "cdn_temp_dns.groupby('cdn')['ip'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cdn_temp_qos_filepath = '~/Desktop/CDN Data/cdn_temp_qos/000004_0'\n",
    "cdn_temp_qos_header = ['cdn', 'cdnip', 'location', 'isp', 'tcp', 'ft',\\\n",
    "                      'mt', 'faultFlag', 'rc', 'url', 'dns', 'ssl', 'dt',\\\n",
    "                      'tt', 'ds', 'avg', 'max', 'min', 'loss',\\\n",
    "                      'ip', 'cname', 'ts']\n",
    "\n",
    "cdn_temp_qos = pd.read_csv(cdn_temp_qos_filepath,\n",
    "                          sep=',', header=None,\n",
    "                          names=cdn_temp_qos_header,\n",
    "                          dtype=str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13\n",
      "['001', '002', '003', '004', '005', '006', '007', '009', '013', '201', '202', '203', '204']\n"
     ]
    }
   ],
   "source": [
    "print cdn_temp_qos['cdn'].nunique()\n",
    "print sorted(cdn_temp_qos['cdn'].unique())\n",
    "# Missing CDN 008, 013"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cdn</th>\n",
       "      <th>cdnip</th>\n",
       "      <th>location</th>\n",
       "      <th>isp</th>\n",
       "      <th>tcp</th>\n",
       "      <th>ft</th>\n",
       "      <th>mt</th>\n",
       "      <th>faultFlag</th>\n",
       "      <th>rc</th>\n",
       "      <th>url</th>\n",
       "      <th>...</th>\n",
       "      <th>dt</th>\n",
       "      <th>tt</th>\n",
       "      <th>ds</th>\n",
       "      <th>avg</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>loss</th>\n",
       "      <th>ip</th>\n",
       "      <th>cname</th>\n",
       "      <th>ts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>001</td>\n",
       "      <td>123.150.53.7</td>\n",
       "      <td>1200</td>\n",
       "      <td>11</td>\n",
       "      <td>8.464</td>\n",
       "      <td>12.299</td>\n",
       "      <td>20150318010035</td>\n",
       "      <td>0</td>\n",
       "      <td>504</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15.811</td>\n",
       "      <td>29.816</td>\n",
       "      <td>62.363</td>\n",
       "      <td>2.767</td>\n",
       "      <td>2.802</td>\n",
       "      <td>2.702</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>001</td>\n",
       "      <td>123.150.53.76</td>\n",
       "      <td>1200</td>\n",
       "      <td>11</td>\n",
       "      <td>8.780</td>\n",
       "      <td>16.784</td>\n",
       "      <td>20150318010035</td>\n",
       "      <td>0</td>\n",
       "      <td>404</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.192</td>\n",
       "      <td>31.786</td>\n",
       "      <td>25.128</td>\n",
       "      <td>2.913</td>\n",
       "      <td>2.923</td>\n",
       "      <td>2.892</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001</td>\n",
       "      <td>123.150.53.77</td>\n",
       "      <td>1200</td>\n",
       "      <td>11</td>\n",
       "      <td>2.991</td>\n",
       "      <td>6.351</td>\n",
       "      <td>20150318010035</td>\n",
       "      <td>0</td>\n",
       "      <td>403</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>6.542</td>\n",
       "      <td>9.599</td>\n",
       "      <td>149.946</td>\n",
       "      <td>4.407</td>\n",
       "      <td>4.442</td>\n",
       "      <td>4.376</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001</td>\n",
       "      <td>123.150.53.78</td>\n",
       "      <td>1200</td>\n",
       "      <td>11</td>\n",
       "      <td>8.768</td>\n",
       "      <td>18.798</td>\n",
       "      <td>20150318010035</td>\n",
       "      <td>0</td>\n",
       "      <td>404</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>19.261</td>\n",
       "      <td>33.405</td>\n",
       "      <td>22.429</td>\n",
       "      <td>2.809</td>\n",
       "      <td>2.846</td>\n",
       "      <td>2.789</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001</td>\n",
       "      <td>123.150.53.79</td>\n",
       "      <td>1200</td>\n",
       "      <td>11</td>\n",
       "      <td>10.325</td>\n",
       "      <td>18.632</td>\n",
       "      <td>20150318010035</td>\n",
       "      <td>0</td>\n",
       "      <td>403</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>19.138</td>\n",
       "      <td>34.987</td>\n",
       "      <td>51.260</td>\n",
       "      <td>4.737</td>\n",
       "      <td>4.78</td>\n",
       "      <td>4.716</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201503</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cdn          cdnip location isp     tcp      ft              mt faultFlag  \\\n",
       "0  001   123.150.53.7     1200  11   8.464  12.299  20150318010035         0   \n",
       "1  001  123.150.53.76     1200  11   8.780  16.784  20150318010035         0   \n",
       "2  001  123.150.53.77     1200  11   2.991   6.351  20150318010035         0   \n",
       "3  001  123.150.53.78     1200  11   8.768  18.798  20150318010035         0   \n",
       "4  001  123.150.53.79     1200  11  10.325  18.632  20150318010035         0   \n",
       "\n",
       "    rc url   ...        dt      tt       ds    avg    max    min loss ip  \\\n",
       "0  504   0   ...    15.811  29.816   62.363  2.767  2.802  2.702    0  0   \n",
       "1  404   0   ...    17.192  31.786   25.128  2.913  2.923  2.892    0  0   \n",
       "2  403   0   ...     6.542   9.599  149.946  4.407  4.442  4.376    0  0   \n",
       "3  404   0   ...    19.261  33.405   22.429  2.809  2.846  2.789    0  0   \n",
       "4  403   0   ...    19.138  34.987   51.260  4.737   4.78  4.716    0  0   \n",
       "\n",
       "  cname      ts  \n",
       "0     0  201503  \n",
       "1     0  201503  \n",
       "2     0  201503  \n",
       "3     0  201503  \n",
       "4     0  201503  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_temp_qos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cdn\n",
       "001    2617\n",
       "002    1673\n",
       "003    1058\n",
       "004     149\n",
       "005     123\n",
       "006      45\n",
       "007       8\n",
       "009      14\n",
       "013      45\n",
       "201     145\n",
       "202      91\n",
       "203      80\n",
       "204     225\n",
       "Name: cdnip, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_temp_qos.groupby('cdn')['cdnip'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cdn_num_ips_df = cdn_num_ips.to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9857.0"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_num_ips_df['ip'].quantile(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_ips_q1 = 0\n",
    "num_ips_q2 = cdn_num_ips_df['ip'].quantile(0.25)\n",
    "num_ips_q3 = cdn_num_ips_df['ip'].quantile(0.50)\n",
    "num_ips_q4 = cdn_num_ips_df['ip'].quantile(0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def num_ips_bin(num_ips):\n",
    "    if num_ips >= num_ips_q1 and num_ips < num_ips_q2: return 'q1'\n",
    "    elif num_ips >= num_ips_q2 and num_ips < num_ips_q3: return 'q2'\n",
    "    elif num_ips >= num_ips_q3 and num_ips < num_ips_q4: return 'q3'\n",
    "    elif num_ips >= num_ips_q4: return 'q4'\n",
    "    else: return 'Error'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for entry in cdn_num_ips_df.itertuples():\n",
    "    cdn = entry[0]\n",
    "    num_ips = entry[1]\n",
    "    ips_bin = num_ips_bin(num_ips)\n",
    "    \n",
    "    if cdn in cdn_to_iidx:\n",
    "        cdn_feature_dict[cdn]['num_ips_bin'] = ips_bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for cdn, features in cdn_feature_dict.iteritems():\n",
    "    features['cdn'] = cdn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'001': {'cdn': '001', 'num_ips_bin': 'q4', 'type': '0'},\n",
       " '002': {'cdn': '002', 'num_ips_bin': 'q4', 'type': '0'},\n",
       " '003': {'cdn': '003', 'num_ips_bin': 'q4', 'type': '0'},\n",
       " '004': {'cdn': '004', 'num_ips_bin': 'q3', 'type': '0'},\n",
       " '005': {'cdn': '005', 'num_ips_bin': 'q4', 'type': '0'},\n",
       " '006': {'cdn': '006', 'num_ips_bin': 'q2', 'type': '0'},\n",
       " '007': {'cdn': '007', 'num_ips_bin': 'q1', 'type': '0'},\n",
       " '008': {'cdn': '008', 'num_ips_bin': 'q1', 'type': '0'},\n",
       " '009': {'cdn': '009', 'num_ips_bin': 'q1', 'type': '0'},\n",
       " '011': {'cdn': '011', 'num_ips_bin': 'q1', 'type': '0'},\n",
       " '013': {'cdn': '013', 'num_ips_bin': 'q2', 'type': '0'},\n",
       " '014': {'cdn': '014', 'type': '0'},\n",
       " '101': {'cdn': '101', 'type': '1'},\n",
       " '102': {'cdn': '102', 'type': '1'},\n",
       " '103': {'cdn': '103', 'type': '1'},\n",
       " '104': {'cdn': '104', 'type': '1'},\n",
       " '105': {'cdn': '105', 'type': '1'},\n",
       " '106': {'cdn': '106', 'type': '1'},\n",
       " '107': {'cdn': '107', 'type': '1'},\n",
       " '108': {'cdn': '108', 'type': '1'},\n",
       " '109': {'cdn': '109', 'type': '1'},\n",
       " '110': {'cdn': '110', 'type': '1'},\n",
       " '111': {'cdn': '111', 'type': '1'},\n",
       " '112': {'cdn': '112', 'type': '1'},\n",
       " '113': {'cdn': '113', 'type': '1'},\n",
       " '114': {'cdn': '114', 'type': '1'},\n",
       " '115': {'cdn': '115', 'type': '1'},\n",
       " '116': {'cdn': '116', 'type': '1'},\n",
       " '117': {'cdn': '117', 'type': '1'},\n",
       " '119': {'cdn': '119', 'type': '1'},\n",
       " '121': {'cdn': '121', 'type': '1'},\n",
       " '122': {'cdn': '122', 'type': '1'},\n",
       " '123': {'cdn': '123', 'type': '1'},\n",
       " '124': {'cdn': '124', 'type': '1'},\n",
       " '125': {'cdn': '125', 'type': '1'},\n",
       " '201': {'cdn': '201', 'num_ips_bin': 'q3', 'type': '2'},\n",
       " '202': {'cdn': '202', 'num_ips_bin': 'q3', 'type': '2'},\n",
       " '203': {'cdn': '203', 'num_ips_bin': 'q2', 'type': '2'},\n",
       " '204': {'cdn': '204', 'num_ips_bin': 'q3', 'type': '2'}}"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_feature_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a list of CDN feature dicts\n",
    "# Ordered by iidx (item/CDN index)\n",
    "cdn_feature_list = [cdn_feature_dict[iidx_to_cdn[iidx]] for iidx in range(len(iidx_to_cdn))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'num_ips_bin': 'q4', 'type': '0', 'cdn': '002'}\n",
      "{'num_ips_bin': 'q3', 'type': '0', 'cdn': '004'}\n",
      "002\n",
      "004\n"
     ]
    }
   ],
   "source": [
    "print cdn_feature_list[0]\n",
    "print cdn_feature_list[1]\n",
    "print iidx_to_cdn[0]\n",
    "print iidx_to_cdn[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Vectorize! (One-hot encodings of each ICP)\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "cdn_vectorizer = DictVectorizer()\n",
    "cdn_feature_vectors = cdn_vectorizer.fit_transform(cdn_feature_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<39x46 sparse matrix of type '<type 'numpy.float64'>'\n",
       "\twith 93 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cdn_feature_vectors\n",
    "# 7 extra features: 4 bins + 3 types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('20170703-cdn-feature-vectors.pkl', 'w') as output:\n",
    "    pickle.dump(cdn_feature_vectors, output, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (RecSys)",
   "language": "python",
   "name": "recsys"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}