API_log_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": true,
        "row": 11,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "source": [
    "Experiment working with anonymized API log data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "%matplotlib inline\n",
    "\n",
    "from bokeh.io import output_notebook\n",
    "from bokeh.plotting import figure, show\n",
    "\n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 5,
        "hidden": true,
        "row": 0,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Import API logs data\n",
    "api_logs = pd.read_csv('api_logs.csv', index_col=0, parse_dates=[0])\n",
    "\n",
    "# Take a look at the columns available\n",
    "api_logs.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 14,
        "hidden": true,
        "row": 11,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Make sure DataFrame has DatetimeIndex\n",
    "print(api_logs.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 17,
        "hidden": false,
        "row": 21,
        "width": 12
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "def is_success_function(row):\n",
    "    \"\"\"\n",
    "    Check if row (request) was successful\n",
    "    Successful means a response with 2xx status\n",
    "    \n",
    "    return True if successful False otherwise\n",
    "    \"\"\"\n",
    "    success = (row['Status'] >= 200 and row['Status'] < 300)\n",
    "    \n",
    "    if success:\n",
    "        return 'Success'\n",
    "    else:\n",
    "        return 'Failure'\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add 'Success' column to API Logs\n",
    "# Success is determined by status code, where any status in the 200s is considered successful\n",
    "api_logs['Outcome'] = api_logs.apply(is_success_function , axis='columns')\n",
    "api_logs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Export CSV containing Outcome column\n",
    "api_logs.to_csv('api_logs_with_outcome.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 4,
        "hidden": true,
        "row": 4,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "source": [
    "# Explore\n",
    "Get a feel for the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 11,
        "hidden": false,
        "row": 10,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Check the distribution of statuses in the data\n",
    "api_response_code_distributions_plot = api_logs['Status'].value_counts().plot('bar', title='Response code counts')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 8,
        "height": 11,
        "hidden": false,
        "row": 10,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Compare number of successful versus unsuccessful requests\n",
    "success_versus_failure = api_logs.groupby('Outcome').size().plot('bar', title='Successful versus unsuccessful requests')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 5,
        "hidden": true,
        "row": 4,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Group API logs by date and outcome\n",
    "daily_failure_success_groups = api_logs.groupby([api_logs.index.date, 'Outcome']).size()\n",
    "\n",
    "# Take a look at the grouped data\n",
    "daily_failure_success_groups.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 7,
        "hidden": true,
        "row": 28,
        "width": 4
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Flatten grouped data, so each row has a failure and success column\n",
    "# Replace NaN numbers with zero (fill_value=0)\n",
    "success_and_failure_over_time = daily_failure_success_groups.unstack(fill_value=0)\n",
    "\n",
    "# Take a look at the unstacked data\n",
    "success_and_failure_over_time.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 4,
        "height": 10,
        "hidden": false,
        "row": 10,
        "width": 4
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Plot success and failure countes over time\n",
    "success_and_failure_chart = success_and_failure_over_time.plot(rot=45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 10,
        "hidden": false,
        "row": 0,
        "width": 12
       },
       "report_default": {
        "hidden": false
       }
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# create a new plot with a datetime axis type\n",
    "p = figure(plot_width=800, plot_height=250, x_axis_type=\"datetime\", title=\"Success and failure over time\")\n",
    "\n",
    "# Add red line showing failure\n",
    "p.line(success_and_failure_over_time.index, success_and_failure_over_time['Failure'], color='red', alpha=0.5)\n",
    "\n",
    "# Add green line showing success\n",
    "p.line(success_and_failure_over_time.index, success_and_failure_over_time['Success'], color='green', alpha=0.5)\n",
    "\n",
    "\n",
    "# Show the plot\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "hidden": true
       },
       "report_default": {
        "hidden": true
       }
      }
     }
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "grid_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}