tutorials/reproducibility.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/phamquiluan/baro/blob/main/tutorials/reproducibility.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IBknvwsqns3l"
      },
      "source": [
        "# Reproduce the RCA performance of BARO\n",
        "\n",
        "In this notebook, we reproduce the effectiveness of BARO on the Online Boutique, Sock Shop, and Train Ticket datasets as presented in Table 3 (The average Avg@5 scores are **0.86**, **0.95**, and **0.81**, respectively)."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install BARO and import packages"
      ],
      "metadata": {
        "id": "Os-9oVHroSYh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q fse-baro"
      ],
      "metadata": {
        "id": "zMXnLlawn50X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bV13VFwNns3m"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import glob\n",
        "import json\n",
        "import warnings\n",
        "from os.path import join, dirname, basename\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from baro.root_cause_analysis import robust_scorer\n",
        "from baro.utility import load_json, download_data"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download three datasets from Zenodo\n",
        "\n"
      ],
      "metadata": {
        "id": "7oaAmaVloGov"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "download_data(\"https://zenodo.org/records/11046533/files/fse-ob.zip?download=1\", \"fse-ob.zip\")\n",
        "import zipfile\n",
        "os.makedirs(\"data\")\n",
        "with zipfile.ZipFile(\"fse-ob.zip\", 'r') as file:\n",
        "    file.extractall(\"data\")\n",
        "\n",
        "download_data(\"https://zenodo.org/records/11046533/files/fse-ss.zip?download=1\", \"fse-ss.zip\")\n",
        "with zipfile.ZipFile(\"fse-ss.zip\", 'r') as file:\n",
        "    file.extractall(\"data\")\n",
        "\n",
        "download_data(\"https://zenodo.org/records/11046533/files/fse-tt.zip?download=1\", \"fse-tt.zip\")\n",
        "with zipfile.ZipFile(\"fse-tt.zip\", 'r') as file:\n",
        "    file.extractall(\"data\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GNbn6MVrn3AZ",
        "outputId": "5c96ffc3-5b10-4d37-8783-1fed9627d3e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Downloading fse-ob.zip..: 100%|██████████| 151M/151M [00:13<00:00, 11.2MiB/s]\n",
            "Downloading fse-ss.zip..: 100%|██████████| 127M/127M [00:11<00:00, 11.4MiB/s]\n",
            "Downloading fse-tt.zip..: 100%|██████████| 286M/286M [00:31<00:00, 9.11MiB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run evaluation on the Online Boutique dataset"
      ],
      "metadata": {
        "id": "Er1-dnjroW9p"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmq_kictns3n",
        "outputId": "00ca1ae7-79a8-4013-9ca9-4208ae59caad"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Avg@5 Accuracy: 0.86\n"
          ]
        }
      ],
      "source": [
        "top1_cnt, top2_cnt, top3_cnt, top4_cnt, top5_cnt, total_cnt = 0, 0, 0, 0, 0, 0\n",
        "\n",
        "for data_path in glob.glob(\"./data/fse-ob/**/simple_data.csv\", recursive=True):\n",
        "    data = pd.read_csv(data_path)\n",
        "    data_dir = os.path.dirname(data_path)\n",
        "\n",
        "    service, metric = basename(dirname(dirname(data_path))).split(\"_\")\n",
        "    # print(f\"{service=} {metric=}\")\n",
        "\n",
        "    ############# PREPROCESSING ###############\n",
        "    if \"time.1\" in data:\n",
        "        data = data.drop(columns=[\"time.1\"])\n",
        "    # handle inf\n",
        "    data = data.replace([np.inf, -np.inf], np.nan)\n",
        "\n",
        "    # handle na\n",
        "    data = data.ffill()\n",
        "    data = data.fillna(0)\n",
        "    # check if there is any nan or inf\n",
        "    if data.isnull().values.any():\n",
        "        print(f\"{data_path=} has nan\")\n",
        "\n",
        "    if data.isin([np.inf, -np.inf]).values.any():\n",
        "        print(f\"{data_path=} has inf\")\n",
        "\n",
        "    data = data.loc[:, ~data.columns.str.endswith(\"latency-50\")]\n",
        "    data = data.rename(\n",
        "        columns={\n",
        "            c: c.replace(\"_latency-90\", \"_latency\")\n",
        "            for c in data.columns\n",
        "            if c.endswith(\"_latency-90\")\n",
        "        }\n",
        "    )\n",
        "\n",
        "    # cut data\n",
        "    data_length = 300\n",
        "    with open(join(data_dir, \"inject_time.txt\")) as f:\n",
        "        inject_time = int(f.readlines()[0].strip())\n",
        "    normal_df = data[data[\"time\"] < inject_time].tail(data_length)\n",
        "    anomal_df = data[data[\"time\"] >= inject_time].head(data_length)\n",
        "    data = pd.concat([normal_df, anomal_df], ignore_index=True)\n",
        "\n",
        "    ############# READ ANOMALY DETECTION OUTPUT ###############\n",
        "    anomalies = load_json(join(data_dir, \"naive_bocpd.json\"))\n",
        "    anomalies = [i[0] for i in anomalies]\n",
        "\n",
        "    ############# ROOT CAUSE ANALYSIS ###############\n",
        "    ranks = robust_scorer(data, anomalies=anomalies)[\"ranks\"]\n",
        "    _service_ranks = [r.split(\"_\")[0] for r in ranks]\n",
        "    service_ranks = []\n",
        "    # remove duplicates\n",
        "    for s in _service_ranks:\n",
        "        if s not in service_ranks:\n",
        "            service_ranks.append(s)\n",
        "\n",
        "    ############## EVALUATION ###############\n",
        "    if service in service_ranks[:1]:\n",
        "        top1_cnt += 1\n",
        "    if service in service_ranks[:2]:\n",
        "        top2_cnt += 1\n",
        "    if service in service_ranks[:3]:\n",
        "        top3_cnt += 1\n",
        "    if service in service_ranks[:4]:\n",
        "        top4_cnt += 1\n",
        "    if service in service_ranks[:5]:\n",
        "        top5_cnt += 1\n",
        "    total_cnt += 1\n",
        "\n",
        "############## EVALUATION ###############\n",
        "top1_accuracy = top1_cnt / total_cnt\n",
        "top2_accuracy = top2_cnt / total_cnt\n",
        "top3_accuracy = top3_cnt / total_cnt\n",
        "top4_accuracy = top4_cnt / total_cnt\n",
        "top5_accuracy = top5_cnt / total_cnt\n",
        "avg5_accuracy = (top1_accuracy + top2_accuracy + top3_accuracy + top4_accuracy + top5_accuracy) / 5\n",
        "\n",
        "print(f\"Avg@5 Accuracy: {avg5_accuracy}\")\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run evaluation on the Sock Shop dataset"
      ],
      "metadata": {
        "id": "ogC-7uNpCLPj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "top1_cnt, top2_cnt, top3_cnt, top4_cnt, top5_cnt, total_cnt = 0, 0, 0, 0, 0, 0\n",
        "\n",
        "for data_path in glob.glob(\"./data/fse-ss/**/simple_data.csv\", recursive=True):\n",
        "    data = pd.read_csv(data_path)\n",
        "    data_dir = os.path.dirname(data_path)\n",
        "\n",
        "    service, metric = basename(dirname(dirname(data_path))).split(\"_\")\n",
        "    # print(f\"{service=} {metric=}\")\n",
        "\n",
        "    ############# PREPROCESSING ###############\n",
        "    if \"time.1\" in data:\n",
        "        data = data.drop(columns=[\"time.1\"])\n",
        "    # handle inf\n",
        "    data = data.replace([np.inf, -np.inf], np.nan)\n",
        "\n",
        "    # handle na\n",
        "    data = data.ffill()\n",
        "    data = data.fillna(0)\n",
        "    # check if there is any nan or inf\n",
        "    if data.isnull().values.any():\n",
        "        print(f\"{data_path=} has nan\")\n",
        "\n",
        "    if data.isin([np.inf, -np.inf]).values.any():\n",
        "        print(f\"{data_path=} has inf\")\n",
        "\n",
        "    data = data.loc[:, ~data.columns.str.endswith(\"latency-50\")]\n",
        "    data = data.rename(\n",
        "        columns={\n",
        "            c: c.replace(\"_latency-90\", \"_latency\")\n",
        "            for c in data.columns\n",
        "            if c.endswith(\"_latency-90\")\n",
        "        }\n",
        "    )\n",
        "\n",
        "    # cut data\n",
        "    data_length = 300\n",
        "    with open(join(data_dir, \"inject_time.txt\")) as f:\n",
        "        inject_time = int(f.readlines()[0].strip())\n",
        "    normal_df = data[data[\"time\"] < inject_time].tail(data_length)\n",
        "    anomal_df = data[data[\"time\"] >= inject_time].head(data_length)\n",
        "    data = pd.concat([normal_df, anomal_df], ignore_index=True)\n",
        "\n",
        "    ############# READ ANOMALY DETECTION OUTPUT ###############\n",
        "    anomalies = load_json(join(data_dir, \"naive_bocpd.json\"))\n",
        "    anomalies = [i[0] for i in anomalies]\n",
        "\n",
        "    ############# ROOT CAUSE ANALYSIS ###############\n",
        "    ranks = robust_scorer(data, anomalies=anomalies)[\"ranks\"]\n",
        "    _service_ranks = [r.split(\"_\")[0] for r in ranks]\n",
        "    service_ranks = []\n",
        "    # remove duplicates\n",
        "    for s in _service_ranks:\n",
        "        if s not in service_ranks:\n",
        "            service_ranks.append(s)\n",
        "\n",
        "    ############## EVALUATION ###############\n",
        "    if service in service_ranks[:1]:\n",
        "        top1_cnt += 1\n",
        "    if service in service_ranks[:2]:\n",
        "        top2_cnt += 1\n",
        "    if service in service_ranks[:3]:\n",
        "        top3_cnt += 1\n",
        "    if service in service_ranks[:4]:\n",
        "        top4_cnt += 1\n",
        "    if service in service_ranks[:5]:\n",
        "        top5_cnt += 1\n",
        "    total_cnt += 1\n",
        "\n",
        "############## EVALUATION ###############\n",
        "top1_accuracy = top1_cnt / total_cnt\n",
        "top2_accuracy = top2_cnt / total_cnt\n",
        "top3_accuracy = top3_cnt / total_cnt\n",
        "top4_accuracy = top4_cnt / total_cnt\n",
        "top5_accuracy = top5_cnt / total_cnt\n",
        "avg5_accuracy = (top1_accuracy + top2_accuracy + top3_accuracy + top4_accuracy + top5_accuracy) / 5\n",
        "\n",
        "print(f\"Avg@5 Accuracy: {avg5_accuracy}\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8ii0uwuBBm6o",
        "outputId": "9362141b-15e0-44a2-9038-5e14d2853631"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Avg@5 Accuracy: 0.95\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run evaluation on the Train Ticket dataset"
      ],
      "metadata": {
        "id": "jHrHnn5vCMvM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "top1_cnt, top2_cnt, top3_cnt, top4_cnt, top5_cnt, total_cnt = 0, 0, 0, 0, 0, 0\n",
        "\n",
        "for data_path in glob.glob(\"./data/fse-tt/**/simple_data.csv\", recursive=True):\n",
        "    data = pd.read_csv(data_path)\n",
        "    data_dir = os.path.dirname(data_path)\n",
        "\n",
        "    service, metric = basename(dirname(dirname(data_path))).split(\"_\")\n",
        "    # print(f\"{service=} {metric=}\")\n",
        "\n",
        "    ############# PREPROCESSING ###############\n",
        "    if \"time.1\" in data:\n",
        "        data = data.drop(columns=[\"time.1\"])\n",
        "    # handle inf\n",
        "    data = data.replace([np.inf, -np.inf], np.nan)\n",
        "\n",
        "    # handle na\n",
        "    data = data.ffill()\n",
        "    data = data.fillna(0)\n",
        "    # check if there is any nan or inf\n",
        "    if data.isnull().values.any():\n",
        "        print(f\"{data_path=} has nan\")\n",
        "\n",
        "    if data.isin([np.inf, -np.inf]).values.any():\n",
        "        print(f\"{data_path=} has inf\")\n",
        "\n",
        "    data = data.loc[:, ~data.columns.str.endswith(\"latency-50\")]\n",
        "    data = data.rename(\n",
        "        columns={\n",
        "            c: c.replace(\"_latency-90\", \"_latency\")\n",
        "            for c in data.columns\n",
        "            if c.endswith(\"_latency-90\")\n",
        "        }\n",
        "    )\n",
        "\n",
        "    # cut data\n",
        "    data_length = 300\n",
        "    with open(join(data_dir, \"inject_time.txt\")) as f:\n",
        "        inject_time = int(f.readlines()[0].strip())\n",
        "    normal_df = data[data[\"time\"] < inject_time].tail(data_length)\n",
        "    anomal_df = data[data[\"time\"] >= inject_time].head(data_length)\n",
        "    data = pd.concat([normal_df, anomal_df], ignore_index=True)\n",
        "\n",
        "    ############# READ ANOMALY DETECTION OUTPUT ###############\n",
        "    anomalies = load_json(join(data_dir, \"naive_bocpd.json\"))\n",
        "    anomalies = [i[0] for i in anomalies]\n",
        "\n",
        "    ############# ROOT CAUSE ANALYSIS ###############\n",
        "    ranks = robust_scorer(data, anomalies=anomalies)[\"ranks\"]\n",
        "    _service_ranks = [r.split(\"_\")[0] for r in ranks]\n",
        "    service_ranks = []\n",
        "    # remove duplicates\n",
        "    for s in _service_ranks:\n",
        "        if s not in service_ranks:\n",
        "            service_ranks.append(s)\n",
        "\n",
        "    ############## EVALUATION ###############\n",
        "    if service in service_ranks[:1]:\n",
        "        top1_cnt += 1\n",
        "    if service in service_ranks[:2]:\n",
        "        top2_cnt += 1\n",
        "    if service in service_ranks[:3]:\n",
        "        top3_cnt += 1\n",
        "    if service in service_ranks[:4]:\n",
        "        top4_cnt += 1\n",
        "    if service in service_ranks[:5]:\n",
        "        top5_cnt += 1\n",
        "    total_cnt += 1\n",
        "\n",
        "############## EVALUATION ###############\n",
        "top1_accuracy = top1_cnt / total_cnt\n",
        "top2_accuracy = top2_cnt / total_cnt\n",
        "top3_accuracy = top3_cnt / total_cnt\n",
        "top4_accuracy = top4_cnt / total_cnt\n",
        "top5_accuracy = top5_cnt / total_cnt\n",
        "avg5_accuracy = (top1_accuracy + top2_accuracy + top3_accuracy + top4_accuracy + top5_accuracy) / 5\n",
        "\n",
        "print(f\"Avg@5 Accuracy: {avg5_accuracy}\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DLloTE5BBsn4",
        "outputId": "f29b74df-739f-4bf0-8295-a1972138829a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Avg@5 Accuracy: 0.806\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "env",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.2"
    },
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}