study/group_importance_study.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting Unstable Software Benchmarks Using Static Source Code Features\n",
    "## Group importance study\n",
    "\n",
    "The following Python Jupyter Notebook can be used to interactively reproduce the study we performed\n",
    "in our paper with the title *Predicting Unstable Software Benchmarks Using Static Source Code Features*.\n",
    "\n",
    "### Initialization\n",
    "\n",
    "We import the needed Python modules."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import itertools\n",
    "\n",
    "from IPython.display import display\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import plotnine as p9\n",
    "import scikit_posthocs as sp\n",
    "from scipy import stats\n",
    "\n",
    "from conf_independent_variables import *\n",
    "from study_conf_labels import *\n",
    "from study_conf_palettes import *\n",
    "from study_data_utils import *\n",
    "from study_plots_utils import *\n",
    "from study_stats_utils import *\n",
    "from utils import apply_binary_threshold, approximate_zeros, remove_negative_values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Configuration\n",
    "\n",
    "First, we configure some parameters for the script.\n",
    "\n",
    "`DATA_CSV_FILE_PATH` specifies the path for the data `CSV` file analyzed bu the notebook.\n",
    "`METRICS` is the list of metrics considered by the study.\n",
    "\n",
    "`BASELINE_MODELS` and `COMPARED_MODELS` differentiate, respectively, the list of models used as baseline and those for comparison.\n",
    "\n",
    "`ITERATIONS` and `THRESHOLDS` represent the values considered for the respective parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GROUP_IMPORTANCE_RESULTS_CSV_FILE_PATH = 'resources/group_importance_mcc_results.csv'\n",
    "PLOTS_OUTPUT_DIRECTORY_PATH = 'resources/output/plots'\n",
    "\n",
    "METRICS = ['precision', 'recall', 'fmeasure', 'auc', 'mcc']\n",
    "\n",
    "BASELINE_MODELS = [\n",
    "    'DummyClassifier(strategy=\\'most_frequent\\')', 'DummyClassifier(strategy=\\'prior\\')',\n",
    "    'DummyClassifier(strategy=\\'stratified\\')', 'DummyClassifier(strategy=\\'uniform\\')',\n",
    "]\n",
    "\n",
    "COMPARED_MODELS = [\n",
    "    'GaussianNB()', 'KNeighborsClassifier()', 'LogisticRegression()', 'MLPClassifier()', 'LinearDiscriminantAnalysis()',\n",
    "    'DecisionTreeClassifier()', 'SVC(kernel=\\'linear\\')', 'SVC(kernel=\\'rbf\\')', 'RandomForestClassifier()',\n",
    "    'AdaBoostClassifier()', 'GradientBoostingClassifier()',\n",
    "]\n",
    "\n",
    "FOCUS_MODELS = ['RandomForestClassifier()']\n",
    "\n",
    "ITERATIONS = [\n",
    "    5,\n",
    "    10,\n",
    "    20,\n",
    "    30,\n",
    "]\n",
    "FOCUS_ITERATIONS = [\n",
    "    30,\n",
    "]\n",
    "\n",
    "THRESHOLDS = [\n",
    "    1,\n",
    "    3,\n",
    "    5,\n",
    "    10,\n",
    "]\n",
    "FOCUS_THRESHOLDS = [\n",
    "    10,\n",
    "]\n",
    "\n",
    "CROSS_VALIDATION_FOLDS = 10\n",
    "CROSS_VALIDATION_REPETITIONS = 30\n",
    "TOTAL_CROSS_VALIDATION_FOLDS = CROSS_VALIDATION_FOLDS * CROSS_VALIDATION_REPETITIONS\n",
    "\n",
    "DEPENDENT_VARIABLES = [\n",
    "    'rciw99',\n",
    "    'rciw99mjhd',\n",
    "    'rmadhd',\n",
    "]\n",
    "FOCUS_DEPENDENT_VARIABLES = [\n",
    "    'rciw99mjhd',\n",
    "]\n",
    "\n",
    "SIGNIFICANCE_LEVEL = 0.01\n",
    "\n",
    "BASELINE_GROUP = IV_GROUP_NONE[0]\n",
    "GROUPS = [\n",
    "    IV_GROUP_NONE[0],\n",
    "    IV_GROUP_BENCH[0],\n",
    "    IV_GROUP_CODE[0],\n",
    "    IV_GROUP_META[0],\n",
    "    IV_GROUP_PL[0],\n",
    "    IV_GROUP_PL_CF[0],\n",
    "    IV_GROUP_PL_DATA[0],\n",
    "    IV_GROUP_PL_CONC[0],\n",
    "    IV_GROUP_LIB[0],\n",
    "    IV_GROUP_IO[0],\n",
    "    IV_GROUP_LIB_CONC[0],\n",
    "    IV_GROUP_MATH[0],\n",
    "    IV_GROUP_STR[0],\n",
    "    IV_GROUP_OS[0],\n",
    "]\n",
    "GROUPS_LABELS = {\n",
    "    IV_GROUP_NONE[0]: 'All features',\n",
    "    IV_GROUP_BENCH[0]: 'bench',\n",
    "    IV_GROUP_CODE[0]: 'code',\n",
    "    IV_GROUP_META[0]: 'meta',\n",
    "    IV_GROUP_PL[0]: 'pl',\n",
    "    IV_GROUP_PL_CF[0]: 'pl cf',\n",
    "    IV_GROUP_PL_DATA[0]: 'pl data',\n",
    "    IV_GROUP_PL_CONC[0]: 'pl conc',\n",
    "    IV_GROUP_LIB[0]: 'lib',\n",
    "    IV_GROUP_IO[0]: 'io',\n",
    "    IV_GROUP_LIB_CONC[0]: 'lib conc',\n",
    "    IV_GROUP_MATH[0]: 'math',\n",
    "    IV_GROUP_STR[0]: 'str',\n",
    "    IV_GROUP_OS[0]: 'os',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Study"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Open the CSV.\n",
    "df = pd.read_csv(GROUP_IMPORTANCE_RESULTS_CSV_FILE_PATH)\n",
    "\n",
    "# Select according to the given configuration.\n",
    "df = df[['dependent_variable', 'iterations', 'threshold', 'model', 'fold', 'excluded_group', *METRICS]]\n",
    "df = df[df['model'].isin(BASELINE_MODELS + COMPARED_MODELS)]\n",
    "df = df[df['iterations'].isin(ITERATIONS)]\n",
    "df = df[df['threshold'].isin(THRESHOLDS)]\n",
    "df = df[df['dependent_variable'].isin(DEPENDENT_VARIABLES)]\n",
    "\n",
    "# Transform some of the columns to categorical type for easy sorting.\n",
    "df['model'] = pd.Categorical(df['model'], categories=BASELINE_MODELS + COMPARED_MODELS)\n",
    "df['iterations'] = pd.Categorical(df['iterations'], categories=ITERATIONS)\n",
    "df['threshold'] = pd.Categorical(df['threshold'], categories=THRESHOLDS)\n",
    "df['dependent_variable'] = pd.Categorical(df['dependent_variable'], categories=DEPENDENT_VARIABLES)\n",
    "df['excluded_group'] = pd.Categorical(df['excluded_group'], categories=GROUPS)\n",
    "\n",
    "# Print the head of the dataframe.\n",
    "display(df)\n",
    "\n",
    "# Print some statistics.\n",
    "print(f\"Number of experiments: {df.shape[0]}\")\n",
    "print(f\"Number of folds per combination: {df['fold'].unique().shape[0]}\")\n",
    "print(f\"Models: {list(df['model'].unique())}\")\n",
    "print(f\"Benchmark iterations: {list(df['iterations'].unique())}\")\n",
    "print(f\"Stability thresholds: {list(df['threshold'].unique())}\")\n",
    "print(f\"Dependent variables: {list(df['dependent_variable'].unique())}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We verify the normality of the distributions by using the *D’Agostino's K^2 Test*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dep_var in FOCUS_DEPENDENT_VARIABLES:\n",
    "    for iterations in FOCUS_ITERATIONS:\n",
    "        for threshold in FOCUS_THRESHOLDS:\n",
    "            for model in FOCUS_MODELS:\n",
    "                # Create a dataframe with the p-values of the normality test.\n",
    "                normal_test_df = pivot_table_grouping(\n",
    "                    df.query('dependent_variable == @dep_var and iterations == @iterations and threshold == @threshold and model == @model'),\n",
    "                    index=['dependent_variable', 'iterations', 'threshold', 'model'],\n",
    "                    columns='excluded_group',\n",
    "                    metrics=METRICS,\n",
    "                    index_sort=[DEPENDENT_VARIABLES, ITERATIONS, THRESHOLDS, BASELINE_MODELS + COMPARED_MODELS],\n",
    "                    columns_sort=[METRICS, GROUPS],\n",
    "                    aggfunc=lambda x: stats.normaltest(x)[1],\n",
    "                )\n",
    "\n",
    "                # Print the dataframe showing the acceptance of the alternative hypothesis as green, and reject as red.\n",
    "                print(f'dep_var={dep_var}, iterations={iterations}, threshold={threshold}, model={model}')\n",
    "                display(normal_test_df\n",
    "                # Rename all the model names into the shortest version.\n",
    "                .rename(index=MODELS_LABELS)\n",
    "                # Show the p-values with reduced decimal digits.\n",
    "                .style.format('{:.4f}')\n",
    "                # Apply the color filtering.\n",
    "                .applymap(lambda x: 'background-color: #5fba7d' if x < SIGNIFICANCE_LEVEL else 'background-color: #d65f5f')\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We test whether there are any statiscally significant differences between the baseline and the comparison by using the *Wilcoxon Test*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dep_var in FOCUS_DEPENDENT_VARIABLES:\n",
    "    for iterations in FOCUS_ITERATIONS:\n",
    "        for threshold in FOCUS_THRESHOLDS:\n",
    "            for model in FOCUS_MODELS:\n",
    "                # Create the dataframe.\n",
    "                wilcoxon_test_df = pairwise_multiple_groups_test_dataframe(\n",
    "                    df.query('dependent_variable == @dep_var and iterations == @iterations and threshold == @threshold and model == @model'),\n",
    "                    group_1=['iterations', 'threshold', 'model'],\n",
    "                    group_2='excluded_group',\n",
    "                    metrics=METRICS,\n",
    "                    testfunc=stats.wilcoxon,\n",
    "                )\n",
    "\n",
    "                # Fix the excluded group and comparison columns.\n",
    "                wilcoxon_test_df['excluded_group'] = pd.Categorical(wilcoxon_test_df['excluded_group'], categories=GROUPS)\n",
    "                wilcoxon_test_df['comparison'] = pd.Categorical(wilcoxon_test_df['comparison'], categories=GROUPS)\n",
    "\n",
    "                # Pivot the dataframe for better visualization.\n",
    "                wilcoxon_test_df = (\n",
    "                    wilcoxon_test_df.pivot_table(index=['iterations', 'threshold', 'model', 'excluded_group'], columns=['metric', 'comparison'], values=['pvalue'])\n",
    "                    # Sort the iterations.\n",
    "                    .reindex(ITERATIONS, level=0)\n",
    "                    # Sort the threshold.\n",
    "                    .reindex(THRESHOLDS, level=1)\n",
    "                    # Sort the models.\n",
    "                    .reindex(BASELINE_MODELS + COMPARED_MODELS, level=2)\n",
    "                    # Sort the excluded groups.\n",
    "                    .reindex(GROUPS, level=3)\n",
    "                    # Sort the metrics.\n",
    "                    .reindex(METRICS, axis=1, level=1)\n",
    "                    # Sort the excluded groups.\n",
    "                    .reindex(GROUPS, axis=1, level=2)\n",
    "                )\n",
    "\n",
    "                # Print the dataframe showing the acceptance of the alternative hypothesis as green, and reject as red.\n",
    "                print(f'dep_var={dep_var}, iterations={iterations}, threshold={threshold}, model={model}')\n",
    "                display(wilcoxon_test_df\n",
    "                # Rename all the model names into the shortest version.\n",
    "                .rename(index=MODELS_LABELS)\n",
    "                # Show the p-values with reduced decimal digits.\n",
    "                .style.format('{:.4f}')\n",
    "                # Apply the color filtering.\n",
    "                .applymap(lambda x: 'background-color: #5fba7d' if x < SIGNIFICANCE_LEVEL else 'background-color: #d65f5f')\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We measure the effect size using the *Vargha-Delaney A* test."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dep_var in FOCUS_DEPENDENT_VARIABLES:\n",
    "    for iterations in FOCUS_ITERATIONS:\n",
    "        for threshold in FOCUS_THRESHOLDS:\n",
    "            for model in FOCUS_MODELS:\n",
    "                # Create the dataframe.\n",
    "                vda_test_df = pairwise_multiple_groups_vda_dataframe(\n",
    "                    df.query('dependent_variable == @dep_var and iterations == @iterations and threshold == @threshold and model == @model'),\n",
    "                    group_1=['iterations', 'threshold', 'model'],\n",
    "                    group_2='excluded_group',\n",
    "                    metrics=METRICS,\n",
    "                )\n",
    "\n",
    "                # Pivot the dataframe for better visualization.\n",
    "                vda_test_df = (\n",
    "                    vda_test_df.pivot_table(index=['iterations', 'threshold', 'model', 'excluded_group'], columns=['metric', 'comparison'], values=['a', 'magnitude'], aggfunc='first')\n",
    "                    # Use \"a\" and \"magnitude\" as an index.\n",
    "                    .stack(level=0)\n",
    "                    # Sort the iterations.\n",
    "                    .reindex(ITERATIONS, level=0)\n",
    "                    # Sort the threshold.\n",
    "                    .reindex(THRESHOLDS, level=1)\n",
    "                    # Sort the models.\n",
    "                    .reindex(BASELINE_MODELS + COMPARED_MODELS, level=2)\n",
    "                    # Sort the excluded groups.\n",
    "                    .reindex(GROUPS, level=3)\n",
    "                    # Sort the metrics.\n",
    "                    .reindex(METRICS, axis=1, level=0)\n",
    "                    # Sort the excluded groups.\n",
    "                    .reindex(GROUPS, axis=1, level=1)\n",
    "                )\n",
    "\n",
    "                # Print the dataframe showing the colored magnitude levels.\n",
    "                print(f'dep_var={dep_var}, iterations={iterations}, threshold={threshold}, model={model}')\n",
    "                display(vda_test_df\n",
    "                # Rename all the model names into the shortest version.\n",
    "                .rename(index=MODELS_LABELS)\n",
    "                # Show all the magnitude values as the shortest version.\n",
    "                .style.format(lambda x: MAGNITUDE_LABELS[x] if isinstance(x, str) else '{:.4f}'.format(x))\n",
    "                # Apply the color filtering.\n",
    "                .applymap(lambda x: f'background-color: {MAGNITUDE_PALETTE[x]}' if x in MAGNITUDE_PALETTE else '')\n",
    "                )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We prepare the data for the comparison."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate the baseline from the comparison instances rows.\n",
    "baseline_group_df = df.query(f'excluded_group == \"{BASELINE_GROUP}\"')\n",
    "comparison_groups_df = df.query(f'excluded_group != \"{BASELINE_GROUP}\"')\n",
    "\n",
    "# Merge on the combinations.\n",
    "merge_df = comparison_groups_df.merge(baseline_group_df, on=['dependent_variable', 'iterations', 'threshold', 'model', 'fold'], suffixes=(None, '_none'))\n",
    "\n",
    "display(merge_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the baseline data to the output dataframe.\n",
    "groups_table_df = pd.concat([baseline_group_df, merge_df], ignore_index=True)\n",
    "\n",
    "for dep_var in FOCUS_DEPENDENT_VARIABLES:\n",
    "    for iterations in FOCUS_ITERATIONS:\n",
    "        for threshold in FOCUS_THRESHOLDS:\n",
    "            for model in FOCUS_MODELS:\n",
    "                focus_median_values_df = groups_table_df.query('dependent_variable == @dep_var and iterations == @iterations and threshold == @threshold and model == @model').groupby(['excluded_group']).median()\n",
    "                # Compute the differences.\n",
    "                for metric in METRICS:\n",
    "                    focus_median_values_df[f'diff_{metric}'] =  focus_median_values_df[f'{metric}'] - focus_median_values_df[f'{metric}_none']\n",
    "                \n",
    "                # Adjust the dataframe.\n",
    "                focus_median_values_df = focus_median_values_df[['fold', *METRICS, *[f'diff_{x}' for x in METRICS]]]\n",
    "\n",
    "                # Sort excluding the baseline row.\n",
    "                focus_median_table_df = focus_median_values_df.loc[~focus_median_values_df.index.isin([BASELINE_GROUP])].sort_values(by=['diff_mcc', 'diff_auc', 'diff_fmeasure', 'diff_precision', 'diff_recall'], ascending=True)\n",
    "                focus_median_table_df = pd.concat([focus_median_values_df.loc[focus_median_values_df.index.isin([BASELINE_GROUP])], focus_median_table_df])\n",
    "\n",
    "                # Print the dataframe showing the bars in the background.\n",
    "                print(f'dep_var={dep_var}, iterations={iterations}, threshold={threshold}, model={model}')\n",
    "                display_columns = list(itertools.chain.from_iterable([(x, f'diff_{x}') for x in METRICS]))\n",
    "                display(\n",
    "                    focus_median_table_df[display_columns]\n",
    "                    # Show the median values with reduced decimal digits.\n",
    "                    .style.format({**{x: '{:.8f}' for x in METRICS}})\n",
    "                    # Show a background bar as indication.\n",
    "                    .bar(vmin=0.0, vmax=1.0, color='#5fba7d')\n",
    "                )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}