main_0_1.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:798fc3153341825ecc6f4efd03a937d4ff7425f80729119ccc5ccac1468a4312"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import pandas as pd\n",
      "import csv\n",
      "from numpy.lib.recfunctions import append_fields\n",
      "import urllib2\n",
      "import StringIO\n",
      "import matplotlib.pyplot as plt\n",
      "import time\n",
      "import math\n",
      "\n",
      "%matplotlib inline "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# TODO\n",
      "\n",
      "1. ~~Programatically load datasets using the quandl API in the prepossessing function~~\n",
      "\n",
      "    - Programatically find the Quandl codes for a given data source (eg: get every data sets from the World Bank)\n",
      "\n",
      "- Function that runs the correlation of a new dataset against all existing datasets\n",
      "    - consider : scalability\n",
      "    - input : new data set\n",
      "    - Correlation: consider using month over month change instead of absolute values (see quandl API call 'Calculations')\n",
      "    - output : list of the \"top10 most correlated data sets\" \n",
      "- Vizualize top 10 list as a tree\n",
      "- function that links all top10 tree, a tree of trees (or graph)\n",
      "- UI (interactive) that combines multiple vizualized trees into a larger linked graph\n",
      "    - might not be possible in ipython\n",
      "    - might be possible using plotly\n",
      "- \n",
      "\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Call Quandl API multiple dataset\n",
      "# the data sets are named using Quandl Codes\n",
      "\n",
      "route = 'FRED/GDP.csv'\n",
      "#     'WORLDBANK.USA_IT_MLT_MAIN,'+\\\n",
      "#     'WORLDBANK.USA_TOTRESV,'+\\\n",
      "#     'WORLDBANK.USA_SP_POP_TOTL,'+\\\n",
      "#     'WORLDBANK.USA_GFDD_OI_05,'+\\\n",
      "#     'WORLDBANK.USA_PANEUATLS,'+\\\n",
      "#     'WORLDBANK.USA_GFDD_OI_04,'+\\\n",
      "#     'WORLDBANK.USA_SM_POP_NETM,'+\\\n",
      "#     'WORLDBANK.USA_SP_URB_TOTL,'+\\\n",
      "#     'WORLDBANK.USA_GFDD_OI_03,'+\\\n",
      "#     'WORLDBANK.USA_SP_RUR_TOTL,'+\\\n",
      "#     'WORLDBANK.USA_SI_POV_GINI,'+\\\n",
      "#     'WORLDBANK.USA_DSTKMKTXN,'+\\\n",
      "#     'WORLDBANK.USA_NY_GDP_MKTP_KN,'+\\\n",
      "#     'WORLDBANK.USA_SP_TER_TOTL_MA_IN,'+\\\n",
      "#     'WORLDBANK.USA_IC_TAX_PAYM,'+\\\n",
      "#     'WORLDBANK.USA_SP_PRM_TOTL_FE_IN,'+\\\n",
      "#     'WORLDBANK.USA_SP_TER_TOTL_IN,'+\\\n",
      "#     'WORLDBANK.USA_SE_PRM_ENRL,'+\\\n",
      "#     'WORLDBANK.USA_SL_TLF_TOTL_FE_IN,WORLDBANK.'+\\\n",
      "#     'USA_EN_BIR_THRD_NO,WORLDBANK.'+\\\n",
      "#     'USA_EN_MAM_THRD_NO,'+\\\n",
      "#     'WORLDBANK.USA_GC_XPN_TOTL_CN'\n",
      "\n",
      "quandlURL = 'https://www.quandl.com/api/v1/datasets/'\n",
      "auth_token = '94CPje6aewK5qpnNArZG'   \n",
      "sort_order = 'desc'\n",
      "start_date = '2005-01-01'\n",
      "end_date = time.strftime('%Y-%m-%d')\n",
      "frequency = 'daily' # collapse=none|daily|weekly|monthly|quarterly|annual\n",
      "calculation = 'none' # rdiff for row-on-row % change, none for abs values\n",
      "\n",
      "URL = quandlURL \\\n",
      "+ route \\\n",
      "+ '?auth_token=' + auth_token \\\n",
      "+ '&trim_start=' + start_date \\\n",
      "+ '&trim_end=' + end_date \\\n",
      "+ '&collapse=' + frequency \\\n",
      "+ '&transformation=' + calculation\n",
      "\n",
      "print URL \n",
      "response = urllib2.urlopen(URL)\n",
      "data = pd.read_csv(response)\n",
      "\n",
      "data"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "https://www.quandl.com/api/v1/datasets/FRED/GDP.csv?auth_token=94CPje6aewK5qpnNArZG&trim_start=2005-01-01&trim_end=2015-01-21&collapse=daily&transformation=none\n"
       ]
      },
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Date</th>\n",
        "      <th>Value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0 </th>\n",
        "      <td> 2014-07-01</td>\n",
        "      <td> 17555.2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td> 2014-04-01</td>\n",
        "      <td> 17328.2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2 </th>\n",
        "      <td> 2014-01-01</td>\n",
        "      <td> 17044.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3 </th>\n",
        "      <td> 2013-10-01</td>\n",
        "      <td> 17078.3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td> 2013-07-01</td>\n",
        "      <td> 16872.3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5 </th>\n",
        "      <td> 2013-04-01</td>\n",
        "      <td> 16619.2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6 </th>\n",
        "      <td> 2013-01-01</td>\n",
        "      <td> 16502.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7 </th>\n",
        "      <td> 2012-10-01</td>\n",
        "      <td> 16332.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8 </th>\n",
        "      <td> 2012-07-01</td>\n",
        "      <td> 16268.9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9 </th>\n",
        "      <td> 2012-04-01</td>\n",
        "      <td> 16094.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td> 2012-01-01</td>\n",
        "      <td> 15956.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td> 2011-10-01</td>\n",
        "      <td> 15785.3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>12</th>\n",
        "      <td> 2011-07-01</td>\n",
        "      <td> 15587.1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>13</th>\n",
        "      <td> 2011-04-01</td>\n",
        "      <td> 15460.9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>14</th>\n",
        "      <td> 2011-01-01</td>\n",
        "      <td> 15238.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>15</th>\n",
        "      <td> 2010-10-01</td>\n",
        "      <td> 15230.2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>16</th>\n",
        "      <td> 2010-07-01</td>\n",
        "      <td> 15057.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>17</th>\n",
        "      <td> 2010-04-01</td>\n",
        "      <td> 14888.6</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>18</th>\n",
        "      <td> 2010-01-01</td>\n",
        "      <td> 14681.1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>19</th>\n",
        "      <td> 2009-10-01</td>\n",
        "      <td> 14566.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
        "      <td> 2009-07-01</td>\n",
        "      <td> 14384.1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>21</th>\n",
        "      <td> 2009-04-01</td>\n",
        "      <td> 14340.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>22</th>\n",
        "      <td> 2009-01-01</td>\n",
        "      <td> 14383.9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>23</th>\n",
        "      <td> 2008-10-01</td>\n",
        "      <td> 14549.9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>24</th>\n",
        "      <td> 2008-07-01</td>\n",
        "      <td> 14843.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25</th>\n",
        "      <td> 2008-04-01</td>\n",
        "      <td> 14813.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>26</th>\n",
        "      <td> 2008-01-01</td>\n",
        "      <td> 14668.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>27</th>\n",
        "      <td> 2007-10-01</td>\n",
        "      <td> 14685.3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>28</th>\n",
        "      <td> 2007-07-01</td>\n",
        "      <td> 14569.7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>29</th>\n",
        "      <td> 2007-04-01</td>\n",
        "      <td> 14422.3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>30</th>\n",
        "      <td> 2007-01-01</td>\n",
        "      <td> 14233.2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>31</th>\n",
        "      <td> 2006-10-01</td>\n",
        "      <td> 14066.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>32</th>\n",
        "      <td> 2006-07-01</td>\n",
        "      <td> 13908.5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>33</th>\n",
        "      <td> 2006-04-01</td>\n",
        "      <td> 13799.8</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>34</th>\n",
        "      <td> 2006-01-01</td>\n",
        "      <td> 13648.9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>35</th>\n",
        "      <td> 2005-10-01</td>\n",
        "      <td> 13381.6</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>36</th>\n",
        "      <td> 2005-07-01</td>\n",
        "      <td> 13205.4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>37</th>\n",
        "      <td> 2005-04-01</td>\n",
        "      <td> 12974.1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>38</th>\n",
        "      <td> 2005-01-01</td>\n",
        "      <td> 12813.7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "          Date    Value\n",
        "0   2014-07-01  17555.2\n",
        "1   2014-04-01  17328.2\n",
        "2   2014-01-01  17044.0\n",
        "3   2013-10-01  17078.3\n",
        "4   2013-07-01  16872.3\n",
        "5   2013-04-01  16619.2\n",
        "6   2013-01-01  16502.4\n",
        "7   2012-10-01  16332.5\n",
        "8   2012-07-01  16268.9\n",
        "9   2012-04-01  16094.7\n",
        "10  2012-01-01  15956.5\n",
        "11  2011-10-01  15785.3\n",
        "12  2011-07-01  15587.1\n",
        "13  2011-04-01  15460.9\n",
        "14  2011-01-01  15238.4\n",
        "15  2010-10-01  15230.2\n",
        "16  2010-07-01  15057.7\n",
        "17  2010-04-01  14888.6\n",
        "18  2010-01-01  14681.1\n",
        "19  2009-10-01  14566.5\n",
        "20  2009-07-01  14384.1\n",
        "21  2009-04-01  14340.4\n",
        "22  2009-01-01  14383.9\n",
        "23  2008-10-01  14549.9\n",
        "24  2008-07-01  14843.0\n",
        "25  2008-04-01  14813.0\n",
        "26  2008-01-01  14668.4\n",
        "27  2007-10-01  14685.3\n",
        "28  2007-07-01  14569.7\n",
        "29  2007-04-01  14422.3\n",
        "30  2007-01-01  14233.2\n",
        "31  2006-10-01  14066.4\n",
        "32  2006-07-01  13908.5\n",
        "33  2006-04-01  13799.8\n",
        "34  2006-01-01  13648.9\n",
        "35  2005-10-01  13381.6\n",
        "36  2005-07-01  13205.4\n",
        "37  2005-04-01  12974.1\n",
        "38  2005-01-01  12813.7"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Compute z-score on all data sources\n",
      "#data_norm = data.ix[:,1:]\n",
      "\n",
      "if calculation == 'rdiff':\n",
      "    ### Doesn't work yet but only need to compute the log of the dataframe data.ix[:,1:] = math.log(data.ix[:,1:])\n",
      "    print 'One minute'\n",
      "else:\n",
      "    data.ix[:,1:] = (data.ix[:,1:]- data.ix[:,1:].mean()) / data.ix[:,1:].std()\n",
      "\n",
      "data.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Date</th>\n",
        "      <th>Value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 2014-07-01</td>\n",
        "      <td> 2.000068</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 2014-04-01</td>\n",
        "      <td> 1.817173</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2014-01-01</td>\n",
        "      <td> 1.588192</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 2013-10-01</td>\n",
        "      <td> 1.615828</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 2013-07-01</td>\n",
        "      <td> 1.449853</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "         Date     Value\n",
        "0  2014-07-01  2.000068\n",
        "1  2014-04-01  1.817173\n",
        "2  2014-01-01  1.588192\n",
        "3  2013-10-01  1.615828\n",
        "4  2013-07-01  1.449853"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Get the top 10 correlated datasets\n",
      "# output (default) is the pearson coefficient\n",
      "\n",
      "# Compute correlation matrix \n",
      "corrMatrix = data.corr()\n",
      "referenceData = 'WORLDBANK.USA_TOTRESV - Value'\n",
      "\n",
      "# Order by max correlation\n",
      "top10 = pd.DataFrame(data=corrMatrix).sort()\n",
      "\n",
      "# Remove the reference data itself\n",
      "# top10 = top10[top10.index != referenceData]\n",
      "\n",
      "# Take top 10\n",
      "top10 = top10.head(10)\n",
      "# top10.value_counts()\n",
      "top10"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Value</th>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 47,
       "text": [
        "       Value\n",
        "Value      1"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# plot the two data sets\n",
      "# Uses z-score to better visualize the correlation\n",
      "\n",
      "plt.plot(d1['Date'], d1['zscore'])\n",
      "plt.plot(d2['Date'], d2['zscore'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "NameError",
       "evalue": "name 'd1' is not defined",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-41-39d3546f6d94>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# Uses z-score to better visualize the correlation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md1\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Date'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0md1\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'zscore'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'Date'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0md2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'zscore'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mNameError\u001b[0m: name 'd1' is not defined"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# plot the data correlation (data set 1 in x, data set 2 in y)\n",
      "\n",
      "plt.scatter(d1['zscore'],d2['zscore'])\n",
      "\n",
      "# adds a trend line\n",
      "# if the data sets where perfectly correlated, the line would fit nicely\n",
      "\n",
      "plt.plot(d1['zscore'],d1['zscore'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "NameError",
       "evalue": "name 'd1' is not defined",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-9-803de4f0b5c7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# plot the data correlation (data set 1 in x, data set 2 in y)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscatter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0md1\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'zscore'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0md2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'zscore'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m# adds a trend line\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mNameError\u001b[0m: name 'd1' is not defined"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}