Conifer_Bitwidth_Profiler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import conifer\n",
    "import datetime\n",
    "import pickle\n",
    "import sklearn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "def open_model(filename):\n",
    "    \"\"\" Returns sklearn stored in pickle file.\"\"\"\n",
    "    with open(filename, 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "\n",
    "def synth_model(model, bit_width='ap_fixed<18,8>'):\n",
    "    \"\"\" Return conifer model given scikit model \"\"\"\n",
    "    # Create a conifer config\n",
    "    cfg = conifer.backends.vivadohls.auto_config()\n",
    "\n",
    "    # Set the output directory to something unique\n",
    "    cfg['OutputDir'] = 'prj_{}'.format(int(datetime.datetime.now().timestamp()))\n",
    "    cfg['XilinxPart'] = 'xc7vx690tffg1761-2'\n",
    "    cfg['Precision'] = bit_width\n",
    "    \n",
    "    # Create and compile the model\n",
    "    conif_model = conifer.model(model, conifer.converters.sklearn, conifer.backends.vhdl, cfg)\n",
    "    conif_model.compile()\n",
    "     \n",
    "    # Synthesize the model\n",
    "    conif_model.build()\n",
    "\n",
    "    return conif_model\n",
    "\n",
    "def load_params_used(filename):\n",
    "    with open(filename, \"r\") as f:\n",
    "        return f.read().split()\n",
    "\n",
    "def load_data(sig_filename, bg_filename, features):\n",
    "    \"\"\" Returns datafram, X, Y for data stored in csv file\"\"\"\n",
    "    \n",
    "    sig_df = pd.read_csv(sig_filename)[features]\n",
    "    bg_df = pd.read_csv(bg_filename)[features]\n",
    "    \n",
    "    sig_df[\"signal\"] = 1\n",
    "    bg_df[\"signal\"] = 0\n",
    "    \n",
    "    df  = pd.concat([sig_df, bg_df], ignore_index=True)\n",
    "    \n",
    "    X = df.to_numpy()[:, :-1]\n",
    "    Y = df.to_numpy()[:, -1][np.newaxis].transpose()\n",
    "    \n",
    "    return df, X, Y\n",
    "\n",
    "def split_data(X, Y, test_percent=0.1):\n",
    "    \"\"\" Returns X_train, Y_train, X_test, Y_test \"\"\"\n",
    "    total = np.concatenate((X, Y), axis=1)\n",
    "    np.random.shuffle(total)\n",
    "    n_X = (int) ((1 - test_percent) * total.shape[0])\n",
    "    \n",
    "    X_train = total[:n_X, :-1]\n",
    "    X_test = total[n_X:, :-1]\n",
    "    \n",
    "    Y_train = total[:n_X, -1][np.newaxis].transpose()\n",
    "    Y_test = total[n_X:, -1][np.newaxis].transpose()\n",
    "    \n",
    "    return X_train, Y_train, X_test, Y_test\n",
    "\n",
    "def get_stats(model, X, Y):\n",
    "    \"\"\" \n",
    "    Returns false positive rates, true positive rates, \n",
    "    and area under curve for given model. \n",
    "    \"\"\"\n",
    "    Y_pred = model.decision_function(X)\n",
    "    fpr, tpr, _ = sklearn.metrics.roc_curve(Y, Y_pred)\n",
    "    auc = sklearn.metrics.auc(fpr, tpr) \n",
    "    return dict(zip([\"fpr\", \"tpr\", \"auc\"], [fpr, tpr, auc]))\n",
    "\n",
    "def make_plot(\n",
    "    results, \n",
    "    colors=['aqua', 'darkorange', 'cornflowerblue', 'red', 'green'],\n",
    "    linewidth=2,\n",
    "    plots_folder='plot.png'\n",
    "    ):\n",
    "    \"\"\"\n",
    "    Results is a dictionary where the key is the name of the model and the value is a\n",
    "    dictionary with the keys fpr, tpr, auc. Plots these results with pretty colors.\n",
    "    \"\"\"\n",
    "    plt.figure()\n",
    " \n",
    "    c = 0\n",
    "    for name, stats in results.items():\n",
    "        plt.plot(stats[\"fpr\"], stats[\"tpr\"], color=colors[c], \n",
    "            lw=linewidth, label=f\"{name} (area = {stats['auc']})\")\n",
    "        c = (c + 1) % len(colors)\n",
    " \n",
    "    #plt.semilogy()\n",
    "    plt.xlim([0.0, 1.0])\n",
    "    #plt.ylim([1e-3, 1.0])\n",
    "    plt.ylim([0.0, 1.0])\n",
    "    plt.xlabel('Signal Efficiency')\n",
    "    plt.ylabel('Background Efficiency')\n",
    "    plt.title('ROC Curves for BDT Tau Lepton Classifier')\n",
    "    plt.legend(loc=\"upper left\")\n",
    "    plt.savefig(plots_folder, bbox_inches='tight')\n",
    "    plt.show()\n",
    "    \n",
    "### int_bit determined by max tree threshold determined\n",
    "### by running conif_model.profile()\n",
    "def get_auc_ratios(grad_model, X_test, Y_test, int_bit, end_bit):\n",
    "    sklearn_stats = get_stats(grad_model, X_test, Y_test) \n",
    "\n",
    "    bit_widths = []\n",
    "    auc_ratios = []\n",
    "    aucs = []\n",
    "\n",
    "    for i in range(int_bit, end_bit):\n",
    "        bw = '<{},{}>'.format(i, int_bit)\n",
    "        bit_widths.append(bw)\n",
    "        conif_model = synth_model(grad_model, 'ap_fixed' + bw)\n",
    "        conif_stats = get_stats(conif_model, X_test, Y_test)\n",
    "        auc_ratios.append(conif_stats[\"auc\"]/sklearn_stats[\"auc\"])\n",
    "        aucs.append(conif_stats[\"auc\"])\n",
    "        \n",
    "    return bit_widths, auc_ratios, aucs\n",
    "\n",
    "def plot_auc_ratios(auc_ratios, bit_widths):\n",
    "    plt.scatter(bit_widths, auc_ratios)\n",
    "    plt.title('Conifer Model Bit Width Profile')\n",
    "    plt.xlabel('Fixed Point Bit Width <Total Bit Width, Integer Width>')\n",
    "    plt.ylabel('Conifer AUC/Sklearn AUC')\n",
    "    plt.xticks(rotation=45)\n",
    "\n",
    "def load_split_data(data=\"./ScaledData\"):\n",
    "    res = []\n",
    "    for i in (\"X_train\", \"Y_train\", \"X_test\", \"Y_test\"):\n",
    "        res.append(np.load(f\"{data}/{i}.npy\"))\n",
    "    return res[0], res[1], res[2], res[3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the scaled model and scaled data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, Y_train, X_test, Y_test = load_split_data()\n",
    "\n",
    "grad_model = open_model('scaled_100_4_model.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine AUC ratios for Sklearn model and Conifer Models of various bitwidths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## change int_bit=10 argument to the best value as determined by \n",
    "## running .profile() on a conifer model (the lower the better).\n",
    "bit_widths, auc_ratios, aucs = get_auc_ratios(grad_model, X_test, Y_test, 10, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the Ratios"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEzCAYAAAAipTTqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAwYklEQVR4nO3deZxcVZn/8c+XEKBZJAgRSRCCgIGoA8EWBRTBLaCjRgYF3COIqLhPfgIug8sIGnVQQRYdQFSIjoYMiBBAkR2hQ4AAEsmwJkFNwCBghBCe3x/nFLkpqqpvh67q213f9+tVr6577vbcpeu595y7KCIwMzPrzzpDHYCZmQ0PThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGGZmVooThrWFpFMkfbHQ/RFJf5H0qKTNhzK2ViSFpB1KDLePpEXPYj4XSnr/2o7fz7S3yet5VJP+x0r6aYvxPyDpqnbE1h9Je0m6M8c/tbiehjIuS5wwupykd0nqy/+gD+R/0Fc92+lGxBER8dU8j9HAd4A3RsTGEfHgIMT9+/zjvktd+excvs+zncezkWN4LK/XZZLOkTSm1j8i9o+IH+dhW/4QSjpE0u11ZZc0KTsqIu7L63lViTgn5FjXHfBCrp7GPZJW5GX9i6QzJG28lpP7CnBijn92cT3Z0HPC6GKSPgOcAHwd2BLYBvgB8LZBntWWwAbAbWszcrMjZeBPwPsKw20OvBJYujbzaYNdImJj4IXAZsCxazmdy4GdJY0FyD/uuwAb1pXtAVzxbINeS2/Jy7ob8HLgC/UDlExK27KW+4m1nxNGl5K0Kelo7mMRMSsiHouIlRFxfkRMz8OsL+kESUvy5wRJ6+d++0haJOmzkv6az06mFaZ/pqSvSXoRsCAXL5f0u9x/p3xE/JCkBZLeWTfuyZJ+I+kxYN8mi/Ez4KBCQjkEOBd4ojCtpsuQ+0/PsS+R9MG6dbS+pG9Jui8fOZ8iqWeg6zoi/g6cB0wqTPv3kg6TtDNwCrBHPkJf3mD8JcBdwN65aDfSj+rldWXrAH31Zw2StpN0uaRHJF0CbFGYfC3BLM/z36MQ47ck/U3S3ZL2L7msi4ELgZfkaYSkj0m6E7gzl31I0sK87c+TNC6X/x8puZ6fY1m/tp4azavVPmTt4YTRvfYgHfWf22KYz5OO2HclHdHuzppHjs8HNgXGA4cCJ0narDiBiPgT8OLcOSYiXitpI+AS4GzgeaQf+h9IenFh1HcB/wlsAjSrrlkC3A68MXe/Dzir7DJI2g/4d+ANwI7A6+vG/QbwojzuDnk5v9QklqbyOpkKXFffLyL+CBwBXJurYcY0mcwVrE4OewNXktZLsey6iHiiwbhnA3NJieKrQLHtpDb+mDz/a3P3K0iJfgvgm8B/S1LrJQVJLwDeBMwrFE/N05sk6bXAccA7ga2Ae4GZABGxPXAf+WwlIh5vMZ8y+5ANMieM7rU5sCwinmwxzLuBr0TEXyNiKfBl4L2F/itz/5UR8RvgUWBiiXn/K3BPRJwREU9GxI3Ar4ADC8P8b0RcHRFPRcQ/W0zrLOB9kiaSfvSurevfahneCZwREbdGxGMUqozyj+OHgE9HxEMR8Qip6u7gEstXc2M+Y1hGqu47dQDj1iueTbyalDCurCu7vH4kSduQqoi+GBGPR8QVwPkl5ndvRPwwt4P8mPTjvmWL4WfnZb0qx/H1Qr/j8jpcQdoep0fEjTkhHE06u5pQIqaiMvuQDbK1buiyYe9BYAtJ67ZIGuNIR4A19+ayp6dRN+4/gDKNndsCr6irflkX+Emh+/4S0wGYBXybtDw/adC/1TKMIx15F/vVjAU2BOYWDqwFNGtPaWS3iFio1Oj/UeBKSZP6SYDNXEE6yt+MdMb07oh4VNJWuexVpPaoeuOAv+WEWHMv8IJ+5vfn2peI+EdeB6227dSIuLRJv+K2HAfcWJj2o5IeJJ293dNPTEVl9iEbZD7D6F7XAv8kVRc0s4T0j1mzTS57tu4HLo+IMYXPxhHxkcIwpR6jHBH/INWZf4TGPxatluEB1vzh3KbwfRmwAnhxIcZNc8PugETESuBHwHbkuv36QUpM464c9+HAfRHxaO51bS7bmAZVXqRl3CxX4dQUl7MTj6suzmON7ZHj2hxYPMBpltmHbJA5YXSpiHiYVB9/ktL17htKGi1pf0nfzIOdA3xB0lhJW+Thm16/PwC/Bl4k6b15nqMlvTw3AK+NY4DXRMQ9Dfq1WoZfAB+QNEnShsB/1EaKiKeAHwL/Jel5AJLGS5oy0OByo/w0UgK6q8EgfwG2lrReP5O6EvhM/ltzVS7ry1U+a4iIe4E+4MuS1lO6ZPothUGWAk+RGps74WxgmqRd88UHXwf+0GTbtTLY+5CV4ITRxSLiO6Qfmy+QfjjuB44EZudBvkb6sbkFmE+qSvjaIMz3EVJD9cGkI84/kxqY1281XovpLYmIZg3jTZchIi4kVeP8DliY/xZ9LpdfJ+nvwKWUa6OpuVnSo8DfSA3Nb4+IhxoM9zvSVU9/lrSsxfQuJzXwFpf1ylzW6nLad5EanR8iJcWnLwzIZ2j/CVwtabmkV/a7VM9CRPwW+CKpveEBYHsG1i5Um86g7kNWjvwCJTMzK8NnGGZmVooThpmZleKEYWZmpThhmJlZKSPqxr0tttgiJkyYMNRhmJkNG3Pnzl0WEWPLDDuiEsaECRPo6+sb6jDMzIYNSff2P1TiKikzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK6VtV0lJOp30kpO/RsQzHumcX1DzXdLbuf4BfCC/BKX2JrTvkt498KOIOL5dcZqZDdTseYuZMWcBS5avYNyYHqZPmcjUyeNHfBztvKz2TOBEnvnKzJr9Sa/F3JH0JM2TSS9EGQWcRHpt5iLgBknnRcTtbYzVrLKq8uNkyex5izl61nxWrFwFwOLlKzh61nyAjm6XoYijbQkjIq7o57WLbwPOivS43OskjZG0FTABWJhfGIOkmXlYJwzrOlX5carFUoXENdRxzJiz4OntUbNi5SpmzFkw4uMYyjaM8az56sZFuaxZeUOSDpfUJ6lv6dKlbQnUbKi0+lHopFriWrx8BcHqxDV73kBflDf841iy/BnvqWpZPpLiGMqEoQZl0aK8oYg4LSJ6I6J37NhSd7ebDRtV+XGqSuKqQhzjxvQMqHwkxTGUCWMRa75PeWvSm7OalZt1nar8OFUlcVUhjulTJtIzetQaZT2jRzF9ykBexjg84xjKhHEe8D4lrwQejogHgBuAHSVtl99xfHAe1qzrVOXHqSqJqwpxTJ08nuMOeCnjx/QgYPyYHo474KUdb88ZijjaeVntOcA+wBaSFpHeJTwaICJOAX5DuqR2Iemy2mm535OSjgTmkC6rPT0ibmtXnGZVVvvnH+rG5ulTJq7R+A5Dd1RdhTimTh5fiSvVOh3HiHqnd29vb/hptWbtMdRXJ1UtjpFC0tyI6C01rBOGmVn3GkjC8KNBzMysFCcMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzMysFCcMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpZd6gDMKuq2fMWM2POApYsX8G4MT1MnzKRqZPHD3VYZkPGCcOsgdnzFnP0rPmsWLkKgMXLV3D0rPkAThrWtVwlZdbAjDkLnk4WNStWrmLGnAVDFJHZ0HPCMGtgyfIVAyo36wZOGGYNjBvTM6Bys27ghGHWwPQpE+kZPWqNsp7Ro5g+ZeIQRWQ29NzobdZArWHbV0mZreaEYdbE1MnjnSDMClwlZWZmpfgMwyrHN8yZVZMThlWKb5gzq662VklJ2k/SAkkLJR3VoP9mks6VdIuk6yW9pNDvHknzJd0kqa+dcVp1+IY5s+pq2xmGpFHAScAbgEXADZLOi4jbC4MdA9wUEW+XtFMe/nWF/vtGxLJ2xWjV4xvmzKqr6RmGpM9IOrRB+cclfarEtHcHFkbEXRHxBDATeFvdMJOA3wJExB3ABElblg3eRh7fMGdWXa2qpD4I/KRB+Wm5X3/GA/cXuhflsqKbgQMAJO0ObAtsnfsFcLGkuZIObzYTSYdL6pPUt3Tp0hJhWZX5hjmz6mqVMCKfGdQXPg6oxLQbDRN13ccDm0m6Cfg4MA94MvfbKyJ2A/YHPiZp7yZBnhYRvRHRO3bs2BJhWZVNnTye4w54KePH9CBg/JgejjvgpW7wNquAlm0YkraMiL/Ul5Wc9iLgBYXurYElxQEi4u/AtDxdAXfnDxGxJP/9q6RzSVVcV5Sctw1jvmHOrJpanWHMAC6Q9BpJm+TPPsD5wLdKTPsGYEdJ20laDzgYOK84gKQxuR/AYcAVEfF3SRtJ2iQPsxHwRuDWgSyYmZkNrqZnGBFxlqSlwFeA2uWutwL/EREX9jfhiHhS0pHAHGAUcHpE3CbpiNz/FGBn4CxJq4DbgVoj+5bAuemkg3WBsyPiorVZQDMzGxyKqG9WGL56e3ujr8+3bJiZlSVpbkT0lhm26RmGpO+zZiN1AMuAyyLiqmcXopmZDTetGr0bHao/F5gh6ecRcUJ7QjIzsypq1Ybx40blkk4BrgFOaFNMZmZWQQN+llRE+BkNZmZdaEDPkpK0LvBe0j0WZmbWRVo1ej/CM+/MXgFcDny4nUGZmVn1tGrD2KSTgZiZWbUNqA1D0vaSviDJd12bmXWZfhOGpK0kfVrS9cBtpLu2D2l7ZGZmVimt3ofxIUm/I7VZbE561tMDEfHliJjfqQDNzKwaWl0ldRJwLfCuiOgDkDRyniNiZmYD0iphjAPeAXwnP9L8F8DojkRlZmaV07RKKiKWRcTJEbE36T3bDwN/lfRHSV/vWIRmZlYJpa6SiohFEfGtiHgZMBV4vK1RmZlZ5QzoTm+AiFgAfLkNsZiZWYUN+FlSZmbWnZwwzMyslFJVUpLGA9sWh4+IK9oVlJmZVU+/CUPSN4CDSO/cXpWLA3DCMDPrImXOMKYCEyPCV0aZmXWxMm0Yd+Eb9szMul6ZM4x/ADdJ+i2F+y8i4hNti8rMzCqnTMI4L3/MzKyLtUwYkkYB742I13coHjMzq6iWbRgRsQr4h6RNOxSPmZlVVJkqqX8C8yVdAjxWK3QbhplZdymTMC7IHzMz62L9JoyI+HEnAjEzs2orc6f3jsBxwCRgg1p5RLywjXGZmVnFlLlx7wzgZOBJYF/gLOAn7QzKzMyqp0zC6ImI3wKKiHsj4ljgte0Ny8zMqqbUVVKS1gHulHQksBh4XnvDMjOzqilzhvEpYEPgE8DLgPcA7y8zcUn7SVogaaGkoxr030zSuZJukXS9pJeUHdfMzDqrzFVSNwBIioiYVnbC+S7xk4A3AIuAGySdFxG3FwY7BrgpIt4uaac8/OtKjmtmZh3U7xmGpD0k3Q78MXfvIukHJaa9O7AwIu6KiCeAmcDb6oaZBPwWICLuACZI2rLkuDbIZs9bzF7H/47tjrqAvY7/HbPnLR7qkMysQspUSZ0ATAEeBIiIm4G9S4w3Hri/0L0olxXdDBwAIGl30lv9ti45Lnm8wyX1SepbunRpibCskdnzFnP0rPksXr6CABYvX8HRs+Y7aZjZ00q90zsi7q8rWtVwwDWp0aTquo8HNpN0E/BxYB7p8t0y49ZiOy0ieiOid+zYsSXCskZmzFnAipVrbtYVK1cxY86CIYrIzKqmzFVS90vaEwhJ65Eav/9YYrxFwAsK3VsDS4oDRMTfgWkAkgTcnT8b9jeuDa4ly1cMqNzMuk+ZM4wjgI+RqoQWAbvm7v7cAOwoabucaA6m7r0aksbkfgCHAVfkJNLvuDa4xo3pGVC5mXWffhNGRCyLiHdHxJYR8byIeE9EPFhivCeBI4E5pDOSX0TEbZKOkHREHmxn4DZJdwD7A59sNe7aLKCVM33KRHpGj1qjrGf0KKZPmThEEZlZ1SiiYdMAkr5Pk3YDqObjzXt7e6Ovr2+owxi2Zs9bzIw5C1iyfAXjxvQwfcpEpk5ueK2BmY0QkuZGRG+ZYVu1YfiXt8tMnTzeCcLMmmqaMGqPNZc0ISLuKfaT9PI2x2VmZhVTptH7V5KePuyU9Brg9PaFZGZmVVT2KqnZkp4v6U3Ad4E3tTcsMzOrmlLPkpL0CeBi0vu93xARvqXazKzLNE0Yks5nzaukNgQeBv5bEhHx1nYHZ2Zm1dHqDONbHYvCzMwqr9VVUpcXuyVtTnro4H0RMbfdgZmZWbU0bfSW9OvaC40kbQXcCnwQ+ImkT3UmPDMzq4pWV0ltFxG35u/TgEsi4i3AK0iJw8zMukirhLGy8P11wG8AIuIR4Kl2BmVmZtXTqtH7fkkfJz2hdjfgIgBJPcDoDsRmZmYV0uoM41DgxcAHgIMiYnkufyVwRnvDMjOzqml1hnEocGpEzCsWRsRlwGVtjcrMzCqnVcK4G/ikpF1I796+ELg4Iv7WkcjMzKxSWt2HMROYCSBpMrAfMEvSKOBS4KKIuL4jUZqZ2ZAr805vcrXUPOA4Sc8B3kB6paoThplZl2h14957JL23Qa+DgPUj4vD2hWVmZlXT6iqpzwKzG5TPzP3MzKyLtEoYo/JNemvIZb4Pw8ysy7RKGKMlbVRfKGkTYL32hWRmZlXUKmH8N/BLSRNqBfn7zNzPzMy6SKvLar8l6VHgckkb5+JHgeMj4uSORGdmZpXR8rLaiDgFOCUnDDVq0zAzs+7Q6hWtn6krCknLgKsi4u72hmVmZlXTqg1jk7rPc4Be4EJJB3cgNjMzq5BWbRhfblQu6bmkR4PMbFdQZmZWPa3OMBqKiIcAtSEWMzOrsAEnDEmvBfzEWjOzLtOq0Xs+EHXFzwWWAO9vZ1BmZlY9rS6r/de67gAejIjH2hiPmZlVVNMqqYi4t+5zH4Ckd0u6oMzEJe0naYGkhZKOatB/U0nnS7pZ0m2SphX63SNpvqSbJPWtzcKZmdng6bcNQ9J6kqZK+gXwAPB64JQS440CTgL2ByYBh0iaVDfYx4DbI2IXYB/g25KKz6naNyJ2jYjeUktjZmZt06oN4w3AIcAU0ju8fwLsHhHTmo1TZ3dgYUTclac3E3gbcHthmAA2kSRgY+Ah4MmBLoSZmbVfqzOMOcD2wKsi4j0RcT7w1ACmPR64v9C9KJcVnQjsTGpInw98MiJq8wjgYklzJTV9WZOkwyX1SepbunTpAMIzM7OBaJUwXgZcB1wq6RJJhwKjBjDtRvdq1F91NQW4CRgH7AqcmF8BC7BXROxGqtL6mKS9G80kIk6LiN6I6B07duwAwjMzs4Fo1eg9LyI+FxHbA8cCk4H1JF3Y6oi/YBHwgkL31qQziaJpwKxIFgJ3Azvl+S/Jf/8KnEuq4jIzsyFS6sa9iLg6Io4kVSmdAOxRYrQbgB0lbZcbsg8Gzqsb5j7gdQCStgQmAndJ2ii/qIn8Eqc3AreWidXMzNqj5ePN6+X2hTn509+wT0o6Mg87Cjg9Im6TdETufwrwVeDMfJOggM9FxDJJLwTOTW3hrAucHREXDSRWMzMbXIqob1YYvnp7e6Ovz7dsmJmVJWlu2VsXBvwsKTMz604tE4akdSS57cDMzPp9RetT+bEd29QeDWLtMXveYmbMWcCS5SsYN6aH6VMmMnVy/W0rZmZDp0yj91bAbZKuB55+8GBEvLVtUXWZ2fMWc/Ss+axYuQqAxctXcPSs+QBOGmZWGWUSRsM379ngmTFnwdPJombFylXMmLPACcPMKqPfhBERl0vaFtgxIi6VtCEDu+Pb+rFk+YoBlZuZDYUyT6v9EPBL4NRcNB6Y3caYus64MT0DKjczGwplLqv9GLAX8HeAiLgTeF47g+o206dMpGf0midtPaNHMX3KxCGKyMzsmcq0YTweEU/ku66RtC7PfIigPQu1dgpfJWVmVVYmYVwu6RigJ78j46PA+e0Nq/tMnTzeCcLMKq1MldRRwFLS+yo+DPwG+EI7gzIzs+pp9ca930bE64DjIuJzwA87F5aZmVVNqyqprSS9Bnhrfr3qGi9Eiogb2xqZmZlVSquE8SVSddTWwHfq+gXw2nYFZWZm1dM0YUTEL4FfSvpiRHy1gzGZmVkFtWrD2Cki7gAukLRbfX9XSZmZdZdWVVKfAQ4Hvt2gn6ukzMy6TKsqqcPz3307F46ZmVVVqXd6S9oTmFAcPiLOalNMZmZWQf0mDEk/AbYHbgJqz+AOwAnDzKyLlDnD6AUmRYSfH2Vm1sXKPBrkVuD57Q7EzMyqrcwZxhbA7fkVrY/XCv2KVjOz7lImYRzb7iDMzKz6yr6idUvg5bno+oj4a3vDMjOzqinzitZ3AtcD7wDeCfxB0oHtDszMzKqlTJXU54GX184qJI0FLiW959vMzLpEmauk1qmrgnqw5HhmZjaClDnDuEjSHOCc3H0QcGH7QjIzsyoq0+g9XdIBwKtIL1E6LSLObXtkZmZWKa0eb74DsGVEXB0Rs4BZuXxvSdtHxP91KkgzMxt6rdoiTgAeaVD+j9yvX5L2k7RA0kJJRzXov6mk8yXdLOk2SdPKjmtmZp3VKmFMiIhb6gsjoo/05NqWJI0CTgL2ByYBh0iaVDfYx4DbI2IXYB/g25LWKzmumZl1UKuEsUGLfj0lpr07sDAi7oqIJ4CZwNvqhglgE0kCNgYeAp4sOa6ZmXVQq4Rxg6QP1RdKOhSYW2La44H7C92LclnRicDOwBJgPvDJiHiq5Li1eA6X1Cepb+nSpSXCMjOztdHqKqlPAedKejerE0QvsB7w9hLTVoOy+kekTyG9Z+O1pHduXCLpypLjpsKI04DTAHp7e/0IdjOzNmn1ita/AHtK2hd4SS6+ICJ+V3Lai4AXFLq3Jp1JFE0Djs/v2lgo6W5gp5LjmplZB5W5D+My4LK1mPYNwI6StgMWAwcD76ob5j7gdcCV+QGHE4G7gOUlxjUzsw4q9U7vtRERT0o6EpgDjAJOj4jbJB2R+58CfBU4U9J8UjXU5yJiGUCjcdsVq5mZ9U8j6c2rvb290dfXN9RhmJkNG5LmRkRvmWH9EEEzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzMysFCcMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSnHCMDOzUpwwzMysFCcMMzMrxQnDzMxKccIwM7NSnDDMzKwUJwwzMyvFCcPMzEpxwjAzs1KcMMzMrJR1hzqAoTZ73mJmzFnAkuUrGDemh+lTJjJ18vihDsvMrHK6OmHMnreYo2fNZ8XKVQAsXr6Co2fNB3DSMDOr09VVUjPmLHg6WdSsWLmKGXMWDFFEZmbV1daEIWk/SQskLZR0VIP+0yXdlD+3Slol6bm53z2S5ud+fe2Ib8nyFQMqNzPrZm1LGJJGAScB+wOTgEMkTSoOExEzImLXiNgVOBq4PCIeKgyyb+7f244Yx43pGVC5mVk3a+cZxu7Awoi4KyKeAGYCb2sx/CHAOW2M5xmmT5lIz+hRa5T1jB7F9CkTOxmGmdmw0M6EMR64v9C9KJc9g6QNgf2AXxWKA7hY0lxJhzebiaTDJfVJ6lu6dOmAApw6eTzHHfBSxo/pQcD4MT0cd8BL3eBtZtZAO6+SUoOyaDLsW4Cr66qj9oqIJZKeB1wi6Y6IuOIZE4w4DTgNoLe3t9n0m5o6ebwThJlZCe08w1gEvKDQvTWwpMmwB1NXHRURS/LfvwLnkqq4zMxsiLQzYdwA7ChpO0nrkZLCefUDSdoUeA3wv4WyjSRtUvsOvBG4tY2xmplZP9pWJRURT0o6EpgDjAJOj4jbJB2R+5+SB307cHFEPFYYfUvgXEm1GM+OiIvaFauZmfVPEQOu9q+s3t7e6Otryy0bZmYjkqS5ZW9d6Oo7vc3MrLwRdYYhaSlw71qOvgWwbBDDWVtViKMKMYDjqOc41lSFOKoQAzy7OLaNiLFlBhxRCePZkNTXrjvKh1scVYjBcTiO4RBHFWLoZByukjIzs1KcMMzMrBQnjNVOG+oAsirEUYUYwHHUcxxrqkIcVYgBOhSH2zDMzKwUn2GYmVkpThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGAOk/Ajd2t9ujcFxOI7hEEcVYhhJcThhlFRbwbH6OuSxkl4m6dBuisFxOI7hEEcVYhiJcfg+jBIkrRMRT+XvzwEOBV4G7Ag8BuwXEU+M9Bgch+MYDnFUIYaRGkc73+k9YkTEU/kNgCcCS0kr+qPAd4HrOrHRqxCD43AcwyGOKsQwUuNwwuhHITvvBawg3YK/hLTSNwFO6oYYHIfjGA5xVCGGkRyHq6SakHQa8DiwPvDViLi/rv9hwHMi4jvFU76RFoPjcBzDIY4qxNANcbjRuwFJXwO2Ak4hvZTkCkmTC/1fDBwFXAfplG8kxuA4HMdwiKMKMXRLHE4YjQk4LyJui4hjSHV9Z0qamPtvCJwcEdfUrj4YoTE4DscxHOKoQgxdEYfbMAokrRsRTwJ3AhMlrR8Rj0fECZLWA2ZL2jsibpB0J6xxmdqIicFxOI7hEEcVYui6OCLCn7oPsA1wJfDp3F1r6/kBsHu3xOA4HMdwiKMKMXRLHD7DACTNAkZHxFsAIuI+SR8GfpZP2c4nZe1dgeeP1Bgch+MYDnFUIYZujaPrr5KS9CngraSrCZZHxJsL/SYCx+TOHYC7IuK9IzEGx+E4hkMcVYihq+Po1GlSVT/AvsAO+fuVwG/q+j8H2ALYu1C2zkiLwXE4juEQRxVi6OY4uv4MA0BST0SsyN+vAB6LiP1z90si4tbCsIo2rLQqxOA4HMdwiKMKMXRrHE4YWeEKAyRdCvwTWAlcFRHf7pYYHIfjaDDfNX5khiKOKsTQJK7uimOwT5GG84fUcFT7/hRwdhvnpaGOoVU83RgHsH5F9o0XVCSO3YY6DtKD8aqwLtYrfK/E/8pQrI+uOsOQ9AXgYeDBiDi7UF58muMo4FhSveAh9f0HIYZeYH5EPF48aqr73tYY8vQ+C/wD+FtEzKzFAKuvze5QHF8EngCWRsTphfJOr48fAzcDP4uIv9TH0ME4vgvcF3VHhUOwPn4IjAUOjoh/1mKAzu0fSo+3OAw4ICJmF8o7vS6+AWwO/LG4XYYgjqNJDw98OCL+p1Desd+vjmfBofoA3wd+CxwM3AF8Hhhb6L8NsGH+/rJC+aA1VAG/Am4A3g9sUNvXOhlDnt4PgQuBDwGLgffV9e9UHKeSLvl7G/AgMHko4sjTvBb4KfBeYKu6ftt2aH38iJSwmvXv1HY5Grig0F1/RD2hA/8rpwI/A44A/hMYXZx+B9fFifn/9tWkg83eIdo3TgTOA95IOrD5BulZUJ1dH4M5sap+gDHABcD2uXsCMBv4IrkagvQUx1/Wjdew2mgtY3g1cCnwGWAG8D5WJ4118t+T2xlDnt57SY8NqHW/BfjeEMTxHuDcQvcPgX8FXlcoa+s2qZvuycCPgbPyOtoIGJf7ndKB9fF20plW7YqXf83b5oBOrw/gC8Cb8/dpwJeBfy/E9oM2/698Fvif/H1XYF7hf3dUB/fRCcAlwPML8zyUNQ/4OrFvbJvjeG7u7gUeAY4qDHNqJ/aNrniWVEQsJ+10+0naJCLuAT4FvAH4RB7mcGADSQcUxhvM+rp5pB+i7wN/Ir3A5CBJG0c+XYyIjwA9bYwB0tnVsYXuh4G98pUWnYzjGtLZHpI+CBxCeuTyCZI+n+fZ7m2CpNr/wEXAL0jP3dmXdAZ2RJ7nEe2OA/gD8CXgaEnfIh3l7wp8N1dFdGR9ZIuAt0iaBhxOOqLdk3RGSkR8tM1xXBUR78jTvQn4HfCNvI+uyuUfAdZv87p4iHTD2xGSvk7aXwV8nFRD0al94xHSNtkhd98JzAEOkfTxPM8P0/71MfLPMFh91HwIcDqwG7kBC3gx0Ae8MHdPImfxNsWybu0vqW72+8CBpIdATu1EDHkeGxS+Pwe4qND9+k7FkefTA/wbsGXu3olUTfbidsfBmtWB+wI/z98vINUVfwjYooPbZStS9cudwNa57EXAfcDOHVgftf+VnYHv5P+X/XPZ5qTq1CkdXB+jC/M6Hdgpd2/QwRgOJh1cXgW8OpdtDdwNvKqDcXwF+Dmpaup60s16k0gHOZt2Ko4Re4Yh6USlB23VjprPIb045DNAr6QNI+I24C+kqgAi4vaIeGiwY6h1R8STuQHqSVJ9+TWkI7e7Saf+kBrWBi2GJnH8s26QUZI2kPQr4N2diiPHsoJUNfWX3H0H6Wj7kdw9qNukGEdERK0hl/So50WSTiXdNXsEsB/w0nbHUeuOiAeAE0jVcoty2Z+AG4FH2x1H4X/lj8AtwAuBN0saGxEPkhJ57ah1UPeP4rooNK6vzL3/RHrC6mdz+T/z305sk5kRcQLph3lMLlsE3EabfjeaxPElUtL8Den9FueR9onHSRevtCWOZ2hnNhqqD+ko7XFS9cuedf0+T6qLPJdU7TBrCGIoHtkuAU5v1K8TcQCbAreT7hI9qZPbpNGyAucAPxqifeOnwNWsPtJ++VDEUTfcTOC/h2h9vDP3v4jU5vfLIYihdiXnlqTE2dFtUtgXDgd+SWrLOY8WFyd0cN/4NfD9dsXRcJ6dnFlHFgg2Bj6c/76PdHq/V90wOwBvpnB10GD+ULeKgdWNdusA+wO/Kow32FdW9BtH/n4D8IOhjINUHfNftDd5ltk31in+7XQchfUxJv9wnDHE66MHeCXwxnbEMYD/lTG0uCejQ+viw6SznP8Yon2jVqW9Iam69KftiqNpfJ2YSac/pHfV1topDs8r/TWF/uvWDd+OyzSHPIYyceTyV1ckjolDHUf9dmnXp0QcIretVWl9tOOHqQoxlIxjnbrhh2qbjGLNm/TaEkfD2Do1o7YvyOqjkY0bbNiPkKpdXgGcSboZaahjOKgC6+Is1rxsc7DPLMrG8WPg3wr9BvuorWwcZ1Rku/wYOLAi62Oo/1eqsk3a9ruxFnG8s137Rr9xdnJmbV+YdPXRecC/1FYmq+tAP0i6XX7mSI/BcTiO4RBHFWJwHAP7jIirpApXuryJdNv8LZCuQ468toHdSQ3ctWv/B3XZqxCD43AcwyGOKsTgONbOiEgYhZXaC8ytrUxJG+a/zwEujIgDc/egPuOlKjE4DscxHOKoQgyOY+2DHREf0lVPVwKb5e7XAz8hn94VhmtbnV8VYnAcjmM4xFGFGBzHwD/D/mm1UnpipKSPkm722hTYDphMenjZ7Eg3HY3oGByH4xgOcVQhBsex9oZ9wgCQtD6wDFhIeozCF0nvt72vm2JwHI5jOMRRhRgcx9oZEQkDQNLOwHrALTFEC1WFGByH4xgOcVQhBscxcCMmYRQNaaNQhWJwHI5jOMRRhRgcRzkjMmGYmdngGxGX1ZqZWfs5YZiZWSlOGGZmVooThpmZleKEYWZmpThhDDJJqyTdVPhMkHTNIE37HklbNCmfL+lmSRdLen6LafRK+l4/8xmT7zxt1r+2jDdLulHSnrl8nKRf5u+7SnpTk/HnSdo1f19X0mOS3lPoP1fSbpK+Iun1DcbfR9KvC9/3LPQ7U9KBrZavP5LOzcu3UNLDhW25Z4Nhp0qaVGKax0r69ybli/P075B0cuFZQk8vv6RP1Z4tVDf+2yTNLnQfLWlhofstks4rbpsG0/i9pN78/ZhC+QRJt/a3bHXT+oCkE/sZZoKkdw1kus+WkmWSNsvdW0kKSa8qDLNU0uaSftRomxaXrX67F9fhSOaEMfhWRMSuhc89EfGMH5o22DcidgH6gGOaDRQRfRHxiX6mNQZomjBYvYy7AEcDx+VpL4n8gDRgV9LTNxupvcscYBdgQa1b0kakd0nfHBFfiohL+4l1n8K0nhVJ60naKCLeHhG7AocBVxa2ZaPEPxXoN2H047/y/CaR3iH+GkjvcS4s/6dIb1qrdw2wR6F7D+Dvkp6Xu/cErq7bNq003XcG0QSg7QlD0rq17/lmuD+wel3tCcxj9X43EVgWEQ9GxGERcXs/k59Kie0uaSNJ661F+JXkhNEBkh7Nf98u6dJ8tLOVpD9Jer6ksZJ+JemG/NkrD795PmOYJ+lUQC1nlFwB7CBpA0ln5DOPeZL2zdMsHp0fK+n0fHR0l6RaIjke2D4f9c7oZ37PAf6WpzdB0q35H+QrwEF5GgfVjXM1q3/k9wROISUYSI9xvjEiVhXPFiTtl4/ArwIOqM0POAL4dJ7Pq/M09pZ0TV6mfn8kJe0s6dukxPWiJsNsK+m3km7Jf7fJZxxvBWbk+W8v6UN5G96ct2mjH/lm1gM2YPX6PFPSgXm7jAMuk3RZcYSIWAo8LGmHXDQe+BVrrt9rimcLknokzczL8nPSa1iRdDzQk5flZ3n8UZJ+KOm2vC/2lF2YHP/3GmyL44FX5/l8WtIoSTPyertF0ofz+OtI+kGe968l/aawP7xM0uVKZ6NzJG2Vy38v6euSLgc+WRdS/X73HdZMINcUplE745qW/08vB2r/l8/Y7nka75B0fR6+ti++CFgg6dtKd3MPb+14omE3f4BVwE35c24ue7TQ/6fAkaQXuB+Sy84GXpW/bwP8MX//HvCl/P3NQABbNJjnPbVy4ETgG6R3D5+Ry3YiPaNmA9IR+a9z+bGkf5L1gS2AB4HRpCPAW0ss4x3Aw8DLcvnT4wEfAE5sMv4E4K78/Zwc32WkV1N+HvhK7ncmcGCO+35gR1LS/EXdMvx7YdpnAv9DOhiaBCxsEsNGwDTgKtIPyWHAJnXDFNfV+cD78/cPkh4K93SMhXE2L3z/GvDxRnEWhjkWWJzX59+As+uW5cD6bdxgGmeS3v88EZgJvA74JumFPH/L66+4bT5Dfnc68C/Ak0Bvg311Qu63a+7+BfCefvb/p7d7s21RXK+5+3DgC/n7+qSz5O3ytv9NHv/5eVkOJO2j1wBj8zgHFZbn9xTeT99ge/4uf7+S9Ha7vtz9Q+CDhWn0kt4zfx8wlpTMr65btuJ2/z3w7fz9TcClhX6bkPavq0n72zRgo6H8nVrbz9OnbDZoVkSqXmjm48CtwHURcU4uez0wSU+/R4XnSNoE2Jt8NB0RF0j6W4vpXiZpFXAL8AXSqy2/n8e9Q9K9ND56viAiHgcel/RXYMuBLKOkPYCzJL2kxHjkeO5Rqv55PilZLABuIL2Ccs9a3AU7AXdHxJ15nj8l/cg0MzvSoxVul9RseR4gravDIuKOEmHvQd4WpMdOf7PJcC+R9DVStd7GwJwS0/6viPiWpNHALyUdHBEzS4xXUztyHgVcC1wPfIn0xNMFEfHPwr4Fab/6HkBE3CLplhbTvjsibsrf55KSyECU2RZvBP6lcAayKeng4FXA/+Tx/1w4u5oIvAS4JC/XKNL2rPl5k/lcD0xWqvYcHRGP5jOfHUjr79t1w78C+H2kszjy2VjDM9BsVv67xnqKiEeAHwG1tpEfAd8lnZ0PK04YnTee9KrFLbX6mTHrAHtExIrigPmfoeyzW/aNiGWFcctUXwE8Xvi+igHuExFxrVJD/NiBjEf6YTsQeCAiQtJ1pFP+3YHrGs1qANMuLlOz9XAgcChwrqRzgB9HxL0DmEezeM4EpkbEzZI+QDqqLTfBiJWSLiL9oA8kYVxDOhAZBfwwIh6RVDubvLrZ7EpOu37/KF0l1WD8ZttCpDOxNZKrpDe3GP62iNijSf/HGhVGxD+ULgj4IHBjLr6OdEbwPNKByzNGazKPRmrL+oz/I0nbks6+DgFuJp1ZDjtuw+ggpUa4M0gNfn8kVQ0AXEyqpqoNt2v+egXw7ly2P7DZAGZXHPdFpKquRv8QjTxCOo3ul6SdSD9U9c/s728aVwOfJiUO8t/3AX+OiOV1w94BbFeoKz5kbWItioiLI+Ig0lHsw8D/KrUvTWgyyjXAwfn7u0lVC43mvwnwQD5bePdAYspJfk/g/xr0brWct5PaOF5NasiFVMV1RI67XnHfeAmpWqpmZY69v1iPlHRkf8M1Ub8sc4CP1OYr6UX5LOAq4N9yW8aWrE6+C4Cx+ewWSaMlvbjkvK8mXUBQ3O8+STrjr08OfwD2UWpLHA28o8UyNJTbji4F/hdYDuwVEQdFxMUl460UJ4zOOoZ01c2VpGRxWG4I+wTQmxv8bif9owN8mdSAeyPptH0gz8f/AanBcj7pFP0DueqpX5Fe2HK1UgN2o0bvWsPoTXna74+IVXXDXEaqZmvU6A3pH/eF5H/ciHiAlHie8QMXEf8kVUFdoNToXTwTOB94u9Zs9C4t0lUx381VbMeQjg4b+QQwLVffvJfVDaozgelKFxZsT3qXwR+AS0iJroxP53V5K+nI9AcNhjkNuFB1jd55GWpXAC2LiJW5+FrS+m2UME4GNs7L8v9IVTXF+dyi1Y3ezezEMw8SyroFeFLpwoBPk6pobgduVGqYP5W0Hn4FLCKtl1NJy/hwRDxBOkP8hqSbScmx7JVya+x3pDONrWm83z1AOhO4FriU1Wcl8Mzt3swq4JhIV9l9Nyr0MqS14afVmtmAKV1pd0D+8W7nfDbObQ2bkxLbXhHx53bO05pzwjCzypL0e9IFBOsB34yIM4cynm7nhGFmZqW4DcPMzEpxwjAzs1KcMMzMrBQnDDMzK8UJw8zMSvn//O3j7FsLYm4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_auc_ratios(auc_ratios, bit_widths)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}