predict_inat21mini.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "78a40757",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pathlib\n",
    "\n",
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "import ml_collections\n",
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "import hier\n",
    "import infer\n",
    "import main\n",
    "import metrics\n",
    "import progmet\n",
    "import tree_util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f4b96da2",
   "metadata": {},
   "outputs": [],
   "source": [
    "hierarchy_file = 'resources/hierarchy/inat21.csv'\n",
    "\n",
    "# config_file = '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-03-31-inat21mini/flat_softmax-lr-0.01-b-64-wd-0.0003-ep-20/config.json'\n",
    "# model_file = '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-03-31-inat21mini/flat_softmax-lr-0.01-b-64-wd-0.0003-ep-20/checkpoints/epoch-0020.pth'\n",
    "\n",
    "config_file = '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-03-31-inat21mini/share_random_cut-0.1-lr-0.01-b-64-wd-0.0003-ep-20/config.json'\n",
    "model_file = '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-03-31-inat21mini/share_random_cut-0.1-lr-0.01-b-64-wd-0.0003-ep-20/checkpoints/epoch-0020.pth'\n",
    "\n",
    "with open(config_file, 'r') as f:\n",
    "    config = ml_collections.ConfigDict(json.load(f))\n",
    "\n",
    "with open(hierarchy_file, 'r') as f:\n",
    "    tree, node_keys = hier.make_hierarchy_from_edges(hier.load_edges(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "93349565",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "num_outputs = main.get_num_outputs(config.predict, tree)\n",
    "net = main.make_model(config.model, num_outputs)\n",
    "missing_keys, unexpected_keys = net.load_state_dict(torch.load(model_file), strict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "054831d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d6cb8a01",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unexpected_keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "52d0666a",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda')\n",
    "net.to(device)\n",
    "loss_fn, pred_fn = main.make_loss(config, tree, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6d05ea75",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset, eval_dataset, tree, _, train_label_map, eval_label_map = main.make_datasets(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c8b00952",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_loader = torch.utils.data.DataLoader(\n",
    "    dataset=eval_dataset,\n",
    "    batch_size=256,\n",
    "    shuffle=False,\n",
    "    num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cce55204",
   "metadata": {},
   "outputs": [],
   "source": [
    "is_leaf = tree.leaf_mask()\n",
    "specificity = -tree.num_leaf_descendants()\n",
    "not_trivial = (tree.num_children() != 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0415e1f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "infer_fns = {\n",
    "    'leaf': lambda p: infer.argmax_where(p, is_leaf),\n",
    "    'majority': lambda p: infer.argmax_with_confidence(specificity, p, 0.5, not_trivial),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0657a1dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "eval:   1% (4/391); T=1.38 f=0.725; mean T=1.38 f=0.725; elapsed 0:00:06; remaining 0:08:53 of 0:08:59\n",
      "eval:   2% (8/391); T=1.39 f=0.719; mean T=1.38 f=0.722; elapsed 0:00:11; remaining 0:08:50 of 0:09:01\n",
      "eval:   3% (12/391); T=1.37 f=0.728; mean T=1.38 f=0.724; elapsed 0:00:17; remaining 0:08:43 of 0:09:00\n",
      "eval:   4% (16/391); T=1.39 f=0.720; mean T=1.38 f=0.723; elapsed 0:00:22; remaining 0:08:39 of 0:09:01\n",
      "eval:   5% (20/391); T=1.39 f=0.721; mean T=1.38 f=0.723; elapsed 0:00:28; remaining 0:08:33 of 0:09:01\n",
      "eval:   6% (24/391); T=1.42 f=0.704; mean T=1.39 f=0.719; elapsed 0:00:33; remaining 0:08:30 of 0:09:04\n",
      "eval:   7% (28/391); T=1.44 f=0.694; mean T=1.40 f=0.716; elapsed 0:00:39; remaining 0:08:27 of 0:09:06\n",
      "eval:   8% (32/391); T=1.45 f=0.689; mean T=1.40 f=0.712; elapsed 0:00:45; remaining 0:08:24 of 0:09:09\n",
      "eval:   9% (36/391); T=1.40 f=0.713; mean T=1.40 f=0.712; elapsed 0:00:51; remaining 0:08:18 of 0:09:09\n",
      "eval:  10% (40/391); T=1.42 f=0.707; mean T=1.41 f=0.712; elapsed 0:00:56; remaining 0:08:13 of 0:09:09\n",
      "eval:  11% (44/391); T=1.41 f=0.707; mean T=1.41 f=0.711; elapsed 0:01:02; remaining 0:08:08 of 0:09:10\n",
      "eval:  12% (48/391); T=1.42 f=0.706; mean T=1.41 f=0.711; elapsed 0:01:08; remaining 0:08:03 of 0:09:10\n",
      "eval:  13% (52/391); T=1.44 f=0.695; mean T=1.41 f=0.710; elapsed 0:01:13; remaining 0:07:58 of 0:09:11\n",
      "eval:  14% (56/391); T=1.39 f=0.721; mean T=1.41 f=0.710; elapsed 0:01:19; remaining 0:07:52 of 0:09:10\n",
      "eval:  15% (60/391); T=1.39 f=0.718; mean T=1.41 f=0.711; elapsed 0:01:24; remaining 0:07:46 of 0:09:10\n",
      "eval:  16% (64/391); T=1.38 f=0.724; mean T=1.41 f=0.712; elapsed 0:01:30; remaining 0:07:39 of 0:09:09\n",
      "eval:  17% (68/391); T=1.40 f=0.714; mean T=1.40 f=0.712; elapsed 0:01:36; remaining 0:07:34 of 0:09:09\n",
      "eval:  18% (72/391); T=1.42 f=0.702; mean T=1.41 f=0.711; elapsed 0:01:41; remaining 0:07:29 of 0:09:10\n",
      "eval:  19% (76/391); T=1.52 f=0.657; mean T=1.41 f=0.708; elapsed 0:01:47; remaining 0:07:25 of 0:09:12\n",
      "eval:  20% (79/391); T=1.38 f=0.725; mean T=1.41 f=0.709; elapsed 0:01:51; remaining 0:07:20 of 0:09:12\n",
      "eval:  21% (83/391); T=1.46 f=0.685; mean T=1.41 f=0.708; elapsed 0:01:57; remaining 0:07:15 of 0:09:13\n",
      "eval:  22% (87/391); T=1.44 f=0.694; mean T=1.41 f=0.707; elapsed 0:02:03; remaining 0:07:10 of 0:09:13\n",
      "eval:  23% (91/391); T=1.45 f=0.691; mean T=1.42 f=0.706; elapsed 0:02:09; remaining 0:07:05 of 0:09:14\n",
      "eval:  24% (95/391); T=1.41 f=0.708; mean T=1.42 f=0.706; elapsed 0:02:15; remaining 0:06:59 of 0:09:14\n",
      "eval:  25% (99/391); T=1.39 f=0.718; mean T=1.41 f=0.707; elapsed 0:02:20; remaining 0:06:53 of 0:09:13\n",
      "eval:  26% (103/391); T=1.42 f=0.703; mean T=1.42 f=0.707; elapsed 0:02:26; remaining 0:06:48 of 0:09:13\n",
      "eval:  27% (107/391); T=1.42 f=0.702; mean T=1.42 f=0.706; elapsed 0:02:31; remaining 0:06:42 of 0:09:13\n",
      "eval:  28% (111/391); T=1.37 f=0.728; mean T=1.41 f=0.707; elapsed 0:02:37; remaining 0:06:36 of 0:09:13\n",
      "eval:  29% (115/391); T=1.37 f=0.729; mean T=1.41 f=0.708; elapsed 0:02:42; remaining 0:06:30 of 0:09:12\n",
      "eval:  30% (119/391); T=1.47 f=0.682; mean T=1.41 f=0.707; elapsed 0:02:48; remaining 0:06:25 of 0:09:13\n",
      "eval:  31% (123/391); T=1.41 f=0.710; mean T=1.41 f=0.707; elapsed 0:02:54; remaining 0:06:19 of 0:09:13\n",
      "eval:  32% (127/391); T=1.38 f=0.725; mean T=1.41 f=0.708; elapsed 0:02:59; remaining 0:06:13 of 0:09:13\n",
      "eval:  34% (131/391); T=1.36 f=0.737; mean T=1.41 f=0.709; elapsed 0:03:05; remaining 0:06:07 of 0:09:12\n",
      "eval:  35% (135/391); T=1.37 f=0.730; mean T=1.41 f=0.709; elapsed 0:03:10; remaining 0:06:01 of 0:09:11\n",
      "eval:  36% (139/391); T=1.39 f=0.719; mean T=1.41 f=0.709; elapsed 0:03:16; remaining 0:05:55 of 0:09:11\n",
      "eval:  37% (143/391); T=1.37 f=0.732; mean T=1.41 f=0.710; elapsed 0:03:21; remaining 0:05:49 of 0:09:11\n",
      "eval:  38% (147/391); T=1.40 f=0.713; mean T=1.41 f=0.710; elapsed 0:03:27; remaining 0:05:44 of 0:09:11\n",
      "eval:  39% (151/391); T=1.44 f=0.693; mean T=1.41 f=0.710; elapsed 0:03:33; remaining 0:05:38 of 0:09:11\n",
      "eval:  40% (155/391); T=1.54 f=0.651; mean T=1.41 f=0.708; elapsed 0:03:39; remaining 0:05:33 of 0:09:12\n",
      "eval:  40% (157/391); T=1.85 f=0.541; mean T=1.42 f=0.705; elapsed 0:03:43; remaining 0:05:32 of 0:09:14\n",
      "eval:  41% (160/391); T=1.95 f=0.513; mean T=1.43 f=0.700; elapsed 0:03:48; remaining 0:05:30 of 0:09:18\n",
      "eval:  42% (164/391); T=1.43 f=0.698; mean T=1.43 f=0.700; elapsed 0:03:54; remaining 0:05:24 of 0:09:18\n",
      "eval:  43% (168/391); T=1.39 f=0.717; mean T=1.43 f=0.701; elapsed 0:04:00; remaining 0:05:18 of 0:09:18\n",
      "eval:  44% (172/391); T=1.35 f=0.743; mean T=1.43 f=0.702; elapsed 0:04:05; remaining 0:05:12 of 0:09:17\n",
      "eval:  45% (176/391); T=1.42 f=0.703; mean T=1.43 f=0.702; elapsed 0:04:11; remaining 0:05:06 of 0:09:17\n",
      "eval:  46% (180/391); T=1.44 f=0.693; mean T=1.43 f=0.701; elapsed 0:04:17; remaining 0:05:01 of 0:09:17\n",
      "eval:  47% (184/391); T=1.43 f=0.700; mean T=1.43 f=0.701; elapsed 0:04:22; remaining 0:04:55 of 0:09:17\n",
      "eval:  48% (188/391); T=1.41 f=0.711; mean T=1.43 f=0.702; elapsed 0:04:28; remaining 0:04:49 of 0:09:17\n",
      "eval:  49% (192/391); T=1.42 f=0.702; mean T=1.43 f=0.702; elapsed 0:04:34; remaining 0:04:44 of 0:09:17\n",
      "eval:  50% (196/391); T=1.42 f=0.703; mean T=1.43 f=0.702; elapsed 0:04:39; remaining 0:04:38 of 0:09:17\n",
      "eval:  51% (200/391); T=1.59 f=0.630; mean T=1.43 f=0.700; elapsed 0:04:46; remaining 0:04:33 of 0:09:19\n",
      "eval:  52% (204/391); T=1.45 f=0.688; mean T=1.43 f=0.700; elapsed 0:04:52; remaining 0:04:27 of 0:09:19\n",
      "eval:  53% (208/391); T=1.40 f=0.713; mean T=1.43 f=0.700; elapsed 0:04:57; remaining 0:04:21 of 0:09:19\n",
      "eval:  54% (212/391); T=1.44 f=0.695; mean T=1.43 f=0.700; elapsed 0:05:03; remaining 0:04:16 of 0:09:19\n",
      "eval:  55% (216/391); T=1.45 f=0.691; mean T=1.43 f=0.700; elapsed 0:05:09; remaining 0:04:10 of 0:09:19\n",
      "eval:  56% (220/391); T=1.42 f=0.704; mean T=1.43 f=0.700; elapsed 0:05:14; remaining 0:04:04 of 0:09:19\n",
      "eval:  57% (224/391); T=1.45 f=0.690; mean T=1.43 f=0.700; elapsed 0:05:20; remaining 0:03:59 of 0:09:19\n",
      "eval:  58% (228/391); T=1.42 f=0.704; mean T=1.43 f=0.700; elapsed 0:05:26; remaining 0:03:53 of 0:09:19\n",
      "eval:  59% (232/391); T=1.43 f=0.698; mean T=1.43 f=0.700; elapsed 0:05:32; remaining 0:03:47 of 0:09:19\n",
      "eval:  60% (235/391); T=1.40 f=0.713; mean T=1.43 f=0.700; elapsed 0:05:36; remaining 0:03:43 of 0:09:19\n",
      "eval:  61% (239/391); T=1.50 f=0.667; mean T=1.43 f=0.699; elapsed 0:05:42; remaining 0:03:37 of 0:09:19\n",
      "eval:  62% (243/391); T=1.53 f=0.656; mean T=1.43 f=0.699; elapsed 0:05:48; remaining 0:03:32 of 0:09:20\n",
      "eval:  63% (247/391); T=1.44 f=0.692; mean T=1.43 f=0.698; elapsed 0:05:54; remaining 0:03:26 of 0:09:20\n",
      "eval:  64% (251/391); T=1.40 f=0.716; mean T=1.43 f=0.699; elapsed 0:05:59; remaining 0:03:20 of 0:09:20\n",
      "eval:  65% (255/391); T=1.46 f=0.683; mean T=1.43 f=0.698; elapsed 0:06:05; remaining 0:03:15 of 0:09:20\n",
      "eval:  66% (259/391); T=1.40 f=0.713; mean T=1.43 f=0.699; elapsed 0:06:11; remaining 0:03:09 of 0:09:20\n",
      "eval:  67% (262/391); T=2.08 f=0.481; mean T=1.44 f=0.695; elapsed 0:06:17; remaining 0:03:06 of 0:09:23\n",
      "eval:  68% (266/391); T=1.42 f=0.703; mean T=1.44 f=0.695; elapsed 0:06:23; remaining 0:03:00 of 0:09:22\n",
      "eval:  69% (270/391); T=1.32 f=0.756; mean T=1.44 f=0.696; elapsed 0:06:28; remaining 0:02:54 of 0:09:22\n",
      "eval:  70% (274/391); T=1.45 f=0.689; mean T=1.44 f=0.696; elapsed 0:06:34; remaining 0:02:48 of 0:09:22\n",
      "eval:  71% (277/391); T=2.05 f=0.488; mean T=1.44 f=0.693; elapsed 0:06:40; remaining 0:02:45 of 0:09:24\n",
      "eval:  72% (281/391); T=1.49 f=0.671; mean T=1.44 f=0.692; elapsed 0:06:46; remaining 0:02:39 of 0:09:25\n",
      "eval:  73% (285/391); T=1.38 f=0.727; mean T=1.44 f=0.693; elapsed 0:06:51; remaining 0:02:33 of 0:09:24\n",
      "eval:  74% (289/391); T=1.38 f=0.726; mean T=1.44 f=0.693; elapsed 0:06:57; remaining 0:02:27 of 0:09:24\n",
      "eval:  75% (293/391); T=1.38 f=0.725; mean T=1.44 f=0.694; elapsed 0:07:02; remaining 0:02:21 of 0:09:24\n",
      "eval:  76% (297/391); T=1.37 f=0.733; mean T=1.44 f=0.694; elapsed 0:07:08; remaining 0:02:15 of 0:09:23\n",
      "eval:  77% (301/391); T=1.35 f=0.740; mean T=1.44 f=0.695; elapsed 0:07:13; remaining 0:02:10 of 0:09:23\n",
      "eval:  78% (305/391); T=1.33 f=0.753; mean T=1.44 f=0.696; elapsed 0:07:19; remaining 0:02:04 of 0:09:22\n",
      "eval:  79% (309/391); T=1.34 f=0.748; mean T=1.44 f=0.696; elapsed 0:07:24; remaining 0:01:58 of 0:09:22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "eval:  80% (313/391); T=1.30 f=0.772; mean T=1.43 f=0.697; elapsed 0:07:29; remaining 0:01:52 of 0:09:21\n",
      "eval:  81% (317/391); T=1.37 f=0.732; mean T=1.43 f=0.697; elapsed 0:07:35; remaining 0:01:46 of 0:09:21\n",
      "eval:  82% (321/391); T=1.33 f=0.750; mean T=1.43 f=0.698; elapsed 0:07:40; remaining 0:01:40 of 0:09:20\n",
      "eval:  83% (325/391); T=1.46 f=0.684; mean T=1.43 f=0.698; elapsed 0:07:46; remaining 0:01:35 of 0:09:20\n",
      "eval:  84% (329/391); T=1.41 f=0.711; mean T=1.43 f=0.698; elapsed 0:07:51; remaining 0:01:29 of 0:09:20\n",
      "eval:  85% (333/391); T=1.45 f=0.690; mean T=1.43 f=0.698; elapsed 0:07:57; remaining 0:01:23 of 0:09:20\n",
      "eval:  86% (337/391); T=1.29 f=0.776; mean T=1.43 f=0.699; elapsed 0:08:02; remaining 0:01:17 of 0:09:20\n",
      "eval:  87% (341/391); T=1.30 f=0.771; mean T=1.43 f=0.700; elapsed 0:08:07; remaining 0:01:11 of 0:09:19\n",
      "eval:  88% (345/391); T=1.26 f=0.793; mean T=1.43 f=0.700; elapsed 0:08:13; remaining 0:01:06 of 0:09:18\n",
      "eval:  90% (350/391); T=1.26 f=0.796; mean T=1.43 f=0.702; elapsed 0:08:19; remaining 0:00:58 of 0:09:17\n",
      "eval:  91% (354/391); T=1.39 f=0.721; mean T=1.42 f=0.702; elapsed 0:08:24; remaining 0:00:53 of 0:09:17\n",
      "eval:  92% (358/391); T=1.33 f=0.755; mean T=1.42 f=0.702; elapsed 0:08:30; remaining 0:00:47 of 0:09:17\n",
      "eval:  93% (362/391); T=1.32 f=0.759; mean T=1.42 f=0.703; elapsed 0:08:35; remaining 0:00:41 of 0:09:16\n",
      "eval:  94% (366/391); T=1.35 f=0.742; mean T=1.42 f=0.703; elapsed 0:08:40; remaining 0:00:36 of 0:09:16\n",
      "eval:  95% (370/391); T=1.46 f=0.684; mean T=1.42 f=0.703; elapsed 0:08:46; remaining 0:00:30 of 0:09:16\n",
      "eval:  96% (374/391); T=1.35 f=0.742; mean T=1.42 f=0.704; elapsed 0:08:52; remaining 0:00:24 of 0:09:16\n",
      "eval:  97% (378/391); T=1.31 f=0.764; mean T=1.42 f=0.704; elapsed 0:08:57; remaining 0:00:18 of 0:09:15\n",
      "eval:  98% (382/391); T=1.30 f=0.768; mean T=1.42 f=0.705; elapsed 0:09:02; remaining 0:00:13 of 0:09:15\n",
      "eval:  99% (386/391); T=1.32 f=0.755; mean T=1.42 f=0.705; elapsed 0:09:07; remaining 0:00:07 of 0:09:14\n",
      "eval: 100% (390/391); T=1.35 f=0.740; mean T=1.42 f=0.706; elapsed 0:09:13; remaining 0:00:01 of 0:09:14\n",
      "eval: 100% (391/391); T=0.774 f=1.29; mean T=1.42 f=0.706; elapsed 0:09:13\n"
     ]
    }
   ],
   "source": [
    "# Per-example predictions.\n",
    "outputs = {\n",
    "    'gt': [],  # Node in hierarchy.\n",
    "    'pred': {method: [] for method in infer_fns},\n",
    "}\n",
    "# Sequence-per-example predictions. Cannot be concatenated due to ragged shape.\n",
    "seq_outputs = {\n",
    "    'pred': [],\n",
    "    'prob': [],\n",
    "}\n",
    "\n",
    "net.eval()\n",
    "with torch.inference_mode():\n",
    "    meter = progmet.ProgressMeter(f'eval', interval_time=5, num_div=5)\n",
    "    for minibatch in meter(eval_loader):\n",
    "        inputs, gt_labels = minibatch\n",
    "        theta = net(inputs.to(device))\n",
    "        prob = pred_fn(theta).cpu().numpy()\n",
    "        pred = {}\n",
    "        for name, infer_fn in infer_fns.items():\n",
    "            pred[name] = infer_fn(prob)\n",
    "        gt_node = eval_label_map.to_node[gt_labels]\n",
    "        pred_seqs = [\n",
    "            infer.pareto_optimal_predictions(specificity, p, 0., not_trivial)\n",
    "            for p in prob\n",
    "        ]\n",
    "        prob_seqs = [prob[i, pred_i] for i, pred_i in enumerate(pred_seqs)]\n",
    "        # Caution: Predictions are *not* truncated.\n",
    "\n",
    "        outputs['gt'].append(gt_node)\n",
    "        for method in infer_fns:\n",
    "            outputs['pred'][method].append(pred[method])\n",
    "        seq_outputs['pred'].extend(pred_seqs)\n",
    "        seq_outputs['prob'].extend(prob_seqs)\n",
    "\n",
    "# Concatenate results from minibatches.\n",
    "leaf_predicate = lambda x: not isinstance(x, dict)  # Treat lists as values, not containers.\n",
    "outputs = tree_util.tree_map(np.concatenate, outputs, is_leaf=leaf_predicate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "65066968",
   "metadata": {},
   "outputs": [],
   "source": [
    "info_metric = metrics.UniformLeafInfoMetric(tree)\n",
    "depth_metric = metrics.DepthMetric(tree)\n",
    "\n",
    "metric_fns = {\n",
    "    'exact': lambda gt, pr: pr == gt,\n",
    "    'correct': metrics.IsCorrect(tree),\n",
    "    'info_excess': info_metric.excess,\n",
    "    'info_deficient': info_metric.deficient,\n",
    "    'info_dist': info_metric.dist,\n",
    "    'info_recall': info_metric.recall,\n",
    "    'info_precision': info_metric.precision,\n",
    "    'depth_excess': depth_metric.excess,\n",
    "    'depth_deficient': depth_metric.deficient,\n",
    "    'depth_dist': depth_metric.dist,\n",
    "    'depth_recall': depth_metric.recall,\n",
    "    'depth_precision': depth_metric.precision,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ccbfbfbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt = outputs['gt']\n",
    "pred = outputs['pred']\n",
    "pred_seq = seq_outputs['pred']\n",
    "prob_seq = seq_outputs['prob']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "28bbd2f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate metrics for each inference method.\n",
    "\n",
    "# First truncate predictions at LCA with ground-truth.\n",
    "# (No impact if gt are all leaf nodes.)\n",
    "pred = {\n",
    "    method: hier.truncate_at_lca(tree, gt, pr)\n",
    "    for method, pr in pred.items()\n",
    "}\n",
    "\n",
    "metric_values = {\n",
    "    method: {field: np.mean(metric_fn(gt, pr))\n",
    "             for field, metric_fn in metric_fns.items()}\n",
    "    for method, pr in pred.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4f22a4bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate metrics for Pareto sequence.\n",
    "find_lca = hier.FindLCA(tree)\n",
    "pred_seq = [hier.truncate_given_lca(gt_i, pr_i, find_lca(gt_i, pr_i)) for gt_i, pr_i in zip(gt, pred_seq)]\n",
    "metric_values_seq = {\n",
    "    field: [metric_fn(gt_i, pr_i) for gt_i, pr_i in zip(gt, pred_seq)]\n",
    "    for field, metric_fn in metric_fns.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f2056c8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "step_scores, step_totals = metrics.operating_curve(prob_seq, metric_values_seq)\n",
    "step_means = {k: v / len(gt) for k, v in step_totals.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4fafdae2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f29d0c6b880>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAlj0lEQVR4nO3deXhU5d3G8e8vC/sOYZEEEiCI7EvYZVNRQAUVF1AroBVRKO4tvl3FtlqttgIi4gJWRUSqVgVFSmUJeyLIKhAgQFAg7HuA5Hn/yGBHGiCQIWcyc3+uay5nzjJzB+Tm4ZwzzzHnHCIiEroivA4gIiKXlopeRCTEqehFREKcil5EJMSp6EVEQlyU1wHOVKVKFRcfH+91DBGRIiU1NXW3cy4mr3VBV/Tx8fGkpKR4HUNEpEgxsy1nW6dDNyIiIU5FLyIS4lT0IiIhTkUvIhLiVPQiIiFORS8iEuJU9CIiIS5kij4nx/GnaWvYvPuI11FERIJKyBR9+p4jfLB0Gz1fnsubyZvJztE8+yIiEEJFXyemDDMf60LHulV45vM13PHaQjZlHvY6loiI50Km6AGqlSvBGwOSeOn2ZqzfeYieL8/jjXmbNLoXkbAWUkUPYGbc0jKWfz/WhU6JVfjjtLXc/tpCNmp0LyJhKuSK/rSq5Urw+j1J/P2O5qTtOkyvl+fx+lyN7kUk/IRs0UPu6P6mFjWZ+WhnOiXG8Kfpa7l13ALSdml0LyLhI6SL/rTc0X0rXu7XnM27j9Br1DzGz92o0b2IhIWwKHrIHd33aV6Trx7tTNf6Mfx5+nca3YtIWAiboj+tatkSvPazVozq34J03+h+3ByN7kUkdIVd0UPu6L53s8v46tEuXHV5VZ774juu+/tcJi3eyrET2V7HExEJKHMuuEaySUlJrjBvJeic48tVO3hldhqrth+kfMlo+rWJ45728dSsULLQcoiIFISZpTrnkvJcF+5Ff5pzjpQt+5gwfzNfrtqBmXFdo2oM6phAUu2KmFmhZxIRya9zFX3Q3RzcK2ZG6/hKtI6vxPb9x/jHwnQmL9nG9JU7aFyzHIM6JHBDsxoUj4r0OqqIyAXRiP4cjp44xcfLtjNxfjobdh2mSpni3NW2Fne1q0XVsiW8jici8iMduikg5xzJabuZMD+d/3y3i2KREfRtFcuDXepSq3Ipr+OJiOjQTUGZGZ0SY+iUGMPm3Ud4Y94mPkzJYErKNvo0u4yHutWlXtWyXscUEcmTRvQXaefB47w+dxPvLd7K8VPZ9GhUnaHd6tG4Znmvo4lIGNKhm0to75ETvJW8mbcXpHMo6xTdLo9h2FX1aFW7ktfRRCSMqOgLwcHjJ3ln4RbemLeJfUdP0r5OZYZdVY8OdSvr0kwRueTOVfT5+masmfUws3VmlmZmI86yze1mtsbMVpvZJL/lA8xsg+8x4OJ+hOBXrkQ0Q7vVY/6Iq/jN9VewMfMwd72xmJvHLmDW2p0E21+oIhI+zjuiN7NIYD3QHcgAlgL9nXNr/LZJBKYAVznn9plZVefcLjOrBKQASYADUoFWzrl9Z/u8ojqiP9Pxk9lMTc1g3JyNZOw7xhU1yjG0W116Nq5BZIRG+CISWAUd0bcB0pxzm5xzJ4DJQJ8ztrkfeOV0gTvndvmWXwfMdM7t9a2bCfS4mB+iqCkRHcnd7Wrz9RNdefG2ZmSdymbYpGV0fv5rXvk6jd2Hs7yOKCJhIj9FXxPY5vc6w7fMX32gvpnNN7NFZtbjAvbFzAabWYqZpWRmZuY/fREQ7bvmfuajXRh3d0tqVy7FCzPW0f7ZWTw8eRlL0/fqsI6IXFKBuo4+CkgEugKxwFwza5LfnZ1z44HxkHvoJkCZgkpkhNGjcQ16NK5B2q7DvLd4C1NTM/jX8u9pUL0sAzrEc3OLmpSI1hQLIhJY+RnRbwfi/F7H+pb5ywA+dc6ddM5tJveYfmI+9w079aqW4fc3NmLx/13Nc7c0IcKMpz5aSftnZ/HXGevYefC41xFFJITk52RsFLnFfTW5Jb0UuNM5t9pvmx7knqAdYGZVgGVAc/57Aralb9NvyD0Zu/dsnxcqJ2MvhHOOxZv38lbyZmau3UmkGTc0rcE9HeJpEVdBl2eKyHkVaAoE59wpMxsGzAAigbecc6vNbCSQ4pz71LfuWjNbA2QDTzrn9vg+/Bly/3IAGHmukg9XZka7OpVpV6cyW/YcYcL8dD5M2cYny7+nTkxp+raM5eYWNblM8+OLyEXQF6aC1KHjJ5m+8gf+mbqdJel7MYMOdStzd9vaXNuoui7RFJGf0Ddji7ite47y0bIMpqZmkLHvGLEVSzKoYwK3J8VStkS01/FEJAio6ENEdo5j5pqdvJm8iaXp+yhbPIoBHeIZ3KUO5VT4ImFNRR+Cvt22n/FzNzFt5Q9ULJU7/cLd7Wrr8kyRMKWiD2ErMw7w/IzvmLdhNzUrlOTR7vW5uUVNHcMXCTMFntRMgleT2PK8c19b3vt5WyqXKcYTH35Lr5fn8eWqHZzKzvE6nogEAY3oQ4hzjukrd/DXr9axefcRapQvwe1JcfRrE0eN8ro0UySU6dBNmDmVncO/1+5i0pKtzNuQSXREBPdemcCwq+pRprjuHikSilT0YWzb3qP8/d8b+Oc3GcSULc7DVydya6tYnbQVCTEqemH5tv088/kaUrfso0qZYgzqmMCgjvGUKqYRvkgoUNEL8N85dcbN2cjsdZlUK1ecx6+9nL4tY3WVjkgRp6tuBPjvnDoTB7Vh6pD21Chfkl9OXcENo5NJ3rDb63gicomo6MNUUnwlPn6oA6P7t+DQ8ZPc/eZifvbmYlZtP+B1NBEJMBV9GDMzbmx2GbMe78Jvrr+CldsPcMPoZIZN+obNu494HU9EAkTH6OVHB4+f5I25m3gjeTNZp3K4o3UcD1+dSLVyJbyOJiLnoZOxckEyD2Ux5j8bmLRkKxFmuROnda5DlTLFvY4mImehopeLsnXPUf7+7/V8vHw7kWZcmViFm1vUpFeTGkRH6qifSDBR0UuBpO06zNTUDD779nu27z9GjfIlGNqtHv3b1NJlmSJBQkUvAZGT45izPpOxs9NYmr6PxjXL8UyfxrSoVdHraCJhT9fRS0BERBjdGlRlygPtGd2/BZmHsrh57AJ+NXUFew5neR1PRM5C33+XC3b6ssxuDaoyatYG3krezPRVPzCwQzyDOiZQqXQxryOKiB8dupEC27DzEC/NXM8Xq3ZQqlgkd7Wtxf2d6lBVl2WKFBodo5dCsWHnIcbO3si/lm8nKjKCfq3jeKBLXWpW0Fz4Ipeail4KVfruI4ybs5F/fpOBc3BLy5o82LUeCVVKex1NJGSp6MUT3+8/xvi5m3h/yVZOZudwQ9PLGNqtHpdXL+t1NJGQo6IXT2UeyuKN5E28u3ALR05k0zahEve0j6dn4+pE6Dp8kYBQ0UtQ2HfkBO8t3sKHqRls2XOU+tXK8PDV9VX4IgFQ4OvozayHma0zszQzG5HH+oFmlmlmy32Pn/uty/Zb/unF/xhS1FUsXYxhVyXyn8e7Mqp/C7JzHEMnfUP3v83hHwvTOZx1yuuIIiHpvCN6M4sE1gPdgQxgKdDfObfGb5uBQJJzblge+x92zpXJbyCN6MNHdo7j8xXf82byZlZkHKBM8ShuS4plUIcEalUu5XU8kSLlXCP6/Hxhqg2Q5pzb5HuzyUAfYM059xI5j8gIo0/zmvRpXpPl2/Yzcf5m3lm4hYkL0ul+RTXuuzKBNgmVMNNhHZGCyM+hm5rANr/XGb5lZ+prZivMbKqZxfktL2FmKWa2yMxuyusDzGywb5uUzMzMfIeX0NE8rgJ/79eC+SOu4qGudVmSvpc7xi/iprELWLJ5r9fxRIq0QM118xkQ75xrCswE3vZbV9v3z4k7gb+bWd0zd3bOjXfOJTnnkmJiYgIUSYqiauVK8OR1DVg44mr+dHNjdh/K4s7XF/H4lG/54cAxr+OJFEn5KfrtgP8IPda37EfOuT3OudOzWr0BtPJbt933303AbKBFAfJKmChZLJK72tbmi0c60a9NHNNX/sC1f5vLi1+t48SpHK/jiRQp+Sn6pUCimSWYWTGgH/CTq2fMrIbfy97AWt/yimZW3Pe8CtARHduXC1CuRDR/vKkJ04ZfyZX1qjD6P2lc89IcpqRsIycnuC4NFglW5y1659wpYBgwg9wCn+KcW21mI82st2+z4Wa22sy+BYYDA33LrwBSfMu/Bp7zv1pHJL/qxJTh1btb8dbAJCqWiuaXU1dw86sLWLhxD8H2XRCRYKMvTEmRk5Pj+GT5dp794jsyD2XRLLY8/9frCtrWqex1NBHP6MYjElIiIoxbWsYy58muPHtLE3YezOKO8YsY/I8UNmUe9jqeSNDRiF6KvGMnsnlr/mZenb2RYyez6Vo/hjtax3HNFdU0tYKEDc11I2Fh9+Es3kzezD9TM9h1KIumvkM67XRIR8KAil7CyqnsHP61/Hv++tU6fjhwnJ6Nq/Nc36aULxntdTSRS0bH6CWsREVG0LdVLF8/0ZUnr7ucmWt20uvlebyzMJ2sU9lexxMpdCp6CVkloiMZ2q0ekwe3I6ZscX77r9VcPyqZLXuOeB1NpFCp6CXkJcVX4uOHOjBhUGt2H87ihlHJfJiyTdffS9hQ0UtYMDO6XV6Vz4ZdSYMaZXly6goen/ItB46e9DqayCWnopewElepFB8Mbs/wqxP5ZPl2rnpxNqNmbWDHgeNeRxO5ZHTVjYSt1d8f4LkvvmPeht1EGFzf9DL+r1cDapQv6XU0kQumyytFzmHLniNMWrKVifPTKRYVwTN9GtOn+WW64YkUKbq8UuQcalcuzVM9r+CrRztTv1pZHvlgOcPeX0bmoazz7yxSBKjoRXxqVy7NlAfa8+R1lzNj1Q66/XU2Hy/L0NU5UuSp6EX8REYYQ7vV46tHO9Ogelke/eBbhrybqtG9FGkqepE81IkpwwcPtOepng34el0m3f82h0+WbdfoXookFb3IWURGGA90qcv04Z1IqFKaRz5Yzt1vLmZpum5WLkWLil7kPOpVLcPUIR34/Y0NWbfjEP3GL2LUrA2czNa9a6VoUNGL5ENkhDGoYwJfP9GVXk1q8NLM9fQeM58VGfu9jiZyXip6kQtQtkQ0o/u3YNzdrdh7JIubxy7ghRnfcfykZsWU4KWiF7kIPRpX56tHu3BT85q88vVG+oyZz6rtB7yOJZInFb3IRSpfMpoXb2/GxEGt2XHwODeMTuaBd1LI2HfU62giP6GiFymgrpdXZe4vu/HINYnM27Cba/82lwnzN5Odo0sxJTio6EUCoHzJaB65pj5fPdqZNgmVePqzNfR9dQHrdhzyOpqIil4kkGIrlmLCwNa83K85W/cepdeoeTz92WqOnjjldTQJYyp6kQAzM/o0r8m/H+vCHa3jmDA/nV4vzyN1yz6vo0mYylfRm1kPM1tnZmlmNiKP9QPNLNPMlvseP/dbN8DMNvgeAwIZXiSYVSpdjD/f3IRJ97flZLbjtnELGPnZGo5kaXQvheu889GbWSSwHugOZABLgf7OuTV+2wwEkpxzw87YtxKQAiQBDkgFWjnnzjq00Xz0EooOHT/Jn6d/xwdLt1KvahlevbsVdWPKeB1LQkhB56NvA6Q55zY5504Ak4E++fzs64CZzrm9vnKfCfTI574iIaNsiWievaUJ/7i3LZmHsrhxdDJfrPzB61gSJvJT9DWBbX6vM3zLztTXzFaY2VQzi7uQfc1ssJmlmFlKZmZmPqOLFD1XJlZh+sOdSKxWlgff+4ZnPl+jyzDlkgvUydjPgHjnXFNyR+1vX8jOzrnxzrkk51xSTExMgCKJBKca5UvyweB23NoqljeTN9Pr5Xl8t+Og17EkhOWn6LcDcX6vY33LfuSc2+OcO31nhjeAVvndVyQclYiO5IVbm/L8rU3J2HeUPmPm89XqHV7HkhCVn6JfCiSaWYKZFQP6AZ/6b2BmNfxe9gbW+p7PAK41s4pmVhG41rdMJOyZGbcnxTHnl924vHpZBr+TysAJS/hmqy7DlMA6b9E7504Bw8gt6LXAFOfcajMbaWa9fZsNN7PVZvYtMBwY6Nt3L/AMuX9ZLAVG+paJiE+VMsWZPLgdT153OSszDnDL2AU88/kaTmm+ewmQ815eWdh0eaWEsyNZp3hhxjomLkinSc3y/PGmxjSLq+B1LCkCCnp5pYgUktLFo/hD70a8cmdLdh48zk1j5/PURyvYdei419GkCFPRiwSh65vWYNbjXbi3YwJTUjK49m9zmZqaoZuTy0VR0YsEqbIlovntDQ2ZNvxK6lQpzRMffkv/1xexff8xr6NJEaOiFwlyDaqXY+qQDjx7SxNWbz/INS/O4Z1FWzS6l3xT0YsUARERRv82tfjXsI60TqjEbz9ZxdOf6Vu1kj8qepEipE5MGSYObM29HROYuCCd+/+RwolTugxTzk1FL1LEREQYv7uxISP7NOI/3+1i0MQl7D6cdf4dJWyp6EWKqHvax/PCrU1Zmr6Pbn+dzftLtnodSYKUil6kCLstKY5Ph3UksWoZnvpoJU99tEK3LZT/oaIXKeIaVC/HlAfac3tSLO8v2Ubn52drgjT5CRW9SAiIiozg+Vub8c8HO1ClTDEGv5PK05+t5rBuWyio6EVCSqvaFflkaEeub1KDCfPTufrF2SxN1zyC4U5FLxJiSkRH8spdLfnngx0oVSyKeycs5dtt+72OJR5S0YuEqFa1K/Lez9tSrmQ0t41byBvzNmnq4zClohcJYZdVKMlnv7iSzvVj+OO0tdz22kK27T3qdSwpZCp6kRBXqXQxXr+nFS/d3oz1Ow5x45hk/rV8u+bKCSMqepEwYGbc0jKWz4d3onq5Ejw8eTm/nLpCc+WECRW9SBhJqFKaacM78bN2tfkwNYOfv72UA0dPeh1LLjEVvUiYiYwwRvZpxNO9GzFvw26uHz1Pl2CGOBW9SBgyMwZ0iGfKkPY4B7eNW8gzn68h61S219HkElDRi4SxlrUqMvOxzvRvE8ebyZu58/XF7DtywutYEmAqepEwV6pYFM/e0pQxd7Zg5fYD9B23QJdghhgVvYgAcEPTy3jn3jbsPpTFLa8uYPX3B7yOJAGioheRH7WtU5mpD3YgKsLoN34R89N2ex1JAkBFLyI/Ub9aWT4c0p7q5Upwz1tL+DBlm9eRpIBU9CLyP2IrluLjoR1pVbsiv/3XKlZm6DBOUZavojezHma2zszSzGzEObbra2bOzJJ8r+PN7JiZLfc9xgUquIhcWmWKRzHmzhZULl2c+95eyvb9x7yOJBfpvEVvZpHAK0BPoCHQ38wa5rFdWeBhYPEZqzY655r7HkMCkFlECknVsiV4/Z4kjmSd4o7XFrIx87DXkeQi5GdE3wZIc85tcs6dACYDffLY7hngL8DxAOYTEY81vKwck+5vx9ET2dwydgFz1md6HUkuUH6KvibgfzYmw7fsR2bWEohzzk3LY/8EM1tmZnPMrFNeH2Bmg80sxcxSMjP1P5FIsGkWV4GPH+pAjfIlGDhhCe8v2ep1JLkABT4Za2YRwEvA43ms/gGo5ZxrATwGTDKzcmdu5Jwb75xLcs4lxcTEFDSSiFwCtSuX5qOHOtC1fgxPfbSScXM2kqPZL4uE/BT9diDO73Wsb9lpZYHGwGwzSwfaAZ+aWZJzLss5twfAOZcKbATqByK4iBS+UsWiGPezVlzfpAbPffEd9729lD2Hs7yOJeeRn6JfCiSaWYKZFQP6AZ+eXumcO+Ccq+Kci3fOxQOLgN7OuRQzi/GdzMXM6gCJwKaA/xQiUmiKR0Uy5s4WjOzTiPlpe+g1ah4LN+7xOpacw3mL3jl3ChgGzADWAlOcc6vNbKSZ9T7P7p2BFWa2HJgKDHHOaT5UkSLOzLinfTwfD+1A6WJR3PXGIsbP3eh1LDkLC7bbiSUlJbmUlBSvY4hIPh3JOsUTH37LF6t20LNxdZ7r25TyJaO9jhV2zCzVOZeU1zp9M1ZECqR08ShG9W/Bw1cn8tWanfR9dQFpuw55HUv8qOhFpMCiIyN4tHt93rm3DfuPnuCG0cmkbtFR2mChoheRgOlQrwofP9SR0sWiuHdiCl+u2uF1JEFFLyIBFlepFFOGtKdWpVIMeTeVpz9bzansHK9jhTUVvYgEXN2YMvzzwQ4MaF+bCfPT+eO0tQTbhR/hJMrrACISmopFRfB0n8acyHZMXJDOlj1HGH1nS8oUV+0UNo3oReSS+tNNjfnDjQ2Zsz6TW19dwI4DmvewsKnoReSSiogwBnZMYMKgNmzbe5Q+ryQzRXetKlQqehEpFF3qxzBlSHuOZGXzy6kreHfRFq8jhQ0VvYgUmkaXlWfpr6+hQfWy/OaTVcxet8vrSGFBRS8ihapksUhevyeJ+tXKcO/Epfx1xjpO6vLLS0pFLyKFLq5SKT4Z2pFrrqjGmK/TePDdVA4cPel1rJClohcRT5QqFsX4e5J4uncj/vPdLq56cbbuSXuJqOhFxFMDOsTz0UMdOXEqhztfX0TaLpV9oKnoRcRzzeMq8OGD7TmZ7fj520s1sg8wFb2IBIUG1cvx+j2t2HPkBL1HJ/Of73Z6HSlkqOhFJGi0ql2Jrx7tTEJMaQb/I5UvVv7gdaSQoKIXkaBSo3xJ3ruvHU1iy/Pge9/w3mJ9saqgVPQiEnTKl4rm/fvb0SmxCr/+eBWjZm3Q7JcFoKIXkaBUIjqScXe34porqvHSzPUMfidVX6y6SCp6EQlapYtH8drPWvHLHpczc81O+oyZz4aduh/thVLRi0hQi4wwHupaj1fvaknGvqNcPyqZrzVHzgVR0YtIkdCzSQ1mPd6VOjGl+cWkZazXyD7fVPQiUmTElC3OhEGtyXGOAW8tYd+RE15HKhJU9CJSpNQoX5KJg9qw5/AJHng3VWWfD/kqejPrYWbrzCzNzEacY7u+ZubMLMlv2VO+/daZ2XWBCC0i4a1NQiVeuK0py7fu54bRyazMOOB1pKB23qI3s0jgFaAn0BDob2YN89iuLPAwsNhvWUOgH9AI6AGM9b2fiEiB9GlekylD2pPjHDeOSSZ5w26vIwWt/Izo2wBpzrlNzrkTwGSgTx7bPQP8BfC/828fYLJzLss5txlI872fiEiBNY+rwNQHOwDw4LupuvH4WeSn6GsC/nfyzfAt+5GZtQTinHPTLnRf3/6DzSzFzFIyMzPzFVxEBKBmhZJMGNiaQ1mnaPfsLA4d1w1MzlTgk7FmFgG8BDx+se/hnBvvnEtyziXFxMQUNJKIhJluDaoyun8LAK55aQ6bNM3xT+Sn6LcDcX6vY33LTisLNAZmm1k60A741HdC9nz7iogExI3NLuP1e5LYeTCL219byIKNOmZ/Wn6KfimQaGYJZlaM3JOrn55e6Zw74Jyr4pyLd87FA4uA3s65FN92/cysuJklAInAkoD/FCIiQPeG1fjooQ5EmHHXG4t5f8lWryMFhfMWvXPuFDAMmAGsBaY451ab2Ugz632efVcDU4A1wJfAUOdcdsFji4jkrWWtikx/uBPNYivw1EcruXnsfE6F+WRoFmxTfyYlJbmUlBSvY4hIEXck6xQ/e3Mx32zdT8/G1RlzZ0siI8zrWJeMmaU655LyWqdvxopISCpdPIqPHurIz9rV5otVO2j0+y/54cAxr2N5QkUvIiFtZJ9GPHnd5Rw/mUPfsQvCcppjFb2IhDQzY2i3egy/OpHvDxznur/P5ctVO7yOVahU9CISFh7rXp9Zj3ehduXSPPReKmNnp4XN7QlV9CISNurGlGHa8Cvp0bg6z3+5joSnppOTE/plr6IXkbBSqlgUr9zZkjLFowAYNHEpR7JOeZzq0lLRi0jYMTNW/uFa7u2YwLwNmfR5ZT5pu0L3JK2KXkTCkpnxuxsb8s59bdl35AQ3j13A9v2hefmlil5EwlrHelX4cEh7sk7m0GfMfNb+cNDrSAGnoheRsFcnpgzv/rwtB4+fpN/4RSzZvNfrSAGlohcRIff2hLMe60KFUtHc/eZiPlkWOhPtquhFRHziKpXiwwfaU6lUMR75YDnxI6aFxLX2KnoRET9Vy5Xgs19c+ePrFs/M5OiJon35pYpeROQMMWWLs/HPvQDYf/QkDX83o0gft1fRi4jkITLCSH/ueoZ0qQvA7a8t5LNvv/c41cVR0YuInMOIng348pFOAPzi/WW8PneTx4kunIpeROQ8GlQvR/KvunF5tbL8afpa4kdM43ARmjZBRS8ikg+xFUvx+fAr6d8mDoDGv5/B4k17PE6VPyp6EZF8io6M4NlbmvLw1YkA3DF+EeuLwI1MVPQiIhfo0e71mTCwNQDX/m1u0J+kVdGLiFyEbg2q/jiy/8X7y3gzebPHic5ORS8icpEe7V6fl/s1B+CZz9fwm09WehvoLFT0IiIF0Kd5TVJ/cw0A7y7aSvyIaUE3A6aKXkSkgCqXKc63v7+WhCqlAej58rygKnsVvYhIAJQvGc3XT3SlS/0YAHqNCp6yV9GLiATQ2/e24cXbmuFc7sj+z9PXeh0pf0VvZj3MbJ2ZpZnZiDzWDzGzlWa23MySzayhb3m8mR3zLV9uZuMC/QOIiASbvq1ifzxJO37uJkbP2uBpnvMWvZlFAq8APYGGQP/TRe5nknOuiXOuOfA88JLfuo3Ouea+x5AA5RYRCWp9mtfk6d6NAHhx5npuHjufjH1HPcmSnxF9GyDNObfJOXcCmAz08d/AOed/IKo0UPRn6hcRKaABHeJ59762VCwVzbKt+7nyL1+zImN/oefIT9HXBLb5vc7wLfsJMxtqZhvJHdEP91uVYGbLzGyOmXXK6wPMbLCZpZhZSmZm5gXEFxEJblcmVmHZ766lV5PqAPQeM5/4EdPYdfB4oWUI2MlY59wrzrm6wK+A3/gW/wDUcs61AB4DJplZuTz2He+cS3LOJcXExAQqkohI0Bh7VyuG+75JC9Dmz7OIHzGNtxekc/xk9iX9bDvf/RDNrD3wB+fcdb7XTwE45549y/YRwD7nXPk81s0GnnDOpZzt85KSklxKyllXi4gUea/P3cT4eZvIPJT1k+WdEqvwzn1tL+o9zSzVOZeU17qofOy/FEg0swRgO9APuPOMD0h0zp0+rXw9sMG3PAbY65zLNrM6QCJQ9GbtFxEJoPs71+H+znU4cOwkk5dsZcHGPcxZn0mz2AqX5PPOW/TOuVNmNgyYAUQCbznnVpvZSCDFOfcpMMzMrgFOAvuAAb7dOwMjzewkkAMMcc4V3RsviogEUPmS0TzQpS4P+G5XeKmc99BNYdOhGxGRC3euQzf6ZqyISIhT0YuIhDgVvYhIiFPRi4iEOBW9iEiIU9GLiIQ4Fb2ISIgLuuvozSwT2OJ1jjNUAXZ7HeIslO3iBGu2YM0FynaxCitbbedcnpOFBV3RByMzSznbFxG8pmwXJ1izBWsuULaLFQzZdOhGRCTEqehFREKcij5/xnsd4ByU7eIEa7ZgzQXKdrE8z6Zj9CIiIU4jehGREKeiFxEJcWFf9GbWw8zWmVmamY3IY/1jZrbGzFaY2Swzq+23boCZbfA9Bpy5rxe5zKy5mS00s9W+dXcEMldBsvmtL2dmGWY2JpiymVktM/vKzNb6tokPomzP+35P15rZKDOzQs42xMxWmtlyM0s2s4Z+657y7bfOzK4LZK6CZDOz7maW6luXamZXBUs2v/W1zOywmT0R6Gw/4ZwL2we5d8zaCNQBigHfAg3P2KYbUMr3/EHgA9/zSuTeFrESUNH3vGIQ5KoPJPqeX0buDdorBMOvmd/6l4FJwJhg+f30vZ4NdPc9L3N6O6+zAR2A+b73iAQWAl0LOVs5v+e9gS99zxv6ti8OJPjeJzJIsrUALvM9bwxs9+D/tzyz+S2bCnxI7r20A5btzEe4j+jbAGnOuU3OuRPAZKCP/wbOua+dc0d9LxcBsb7n1wEznXN7nXP7gJlAD69zOefWO9/9e51z3wO7gDy/LVfY2QDMrBVQDfgqgJkKnM030opyzs30bXfYbztPswEOKEFumRQHooGdhZztoN/L0r5M+Lab7JzLcs5tBtJ87+d5NufcMt+fAYDVQEkzKx4M2QDM7CZgsy/bJRXuRV8T2Ob3OsO37GzuA764yH0LK9ePzKwNueWwMUC5CpTNzCKAF4FL9c/Ugvy61Qf2m9lHZrbMzF4ws8hgyOacWwh8Te6/zn4AZjjn1hZ2NjMbamYbgeeB4Reyr0fZ/PUFvnHOZQVDNjMrA/wKeDqAec4q3Is+38zsbiAJeMHrLP7OlsvMagDvAIOcczlBku0hYLpzLsOLPP7yyBYFdCL3L6HW5P5zfGAwZDOzesAV5I7wawJXmVmnws7lnHvFOVeX3IL6TWF//rmcK5uZNQL+AjwQRNn+APzNOXe4MDJEFcaHBLHtQJzf61jfsp8ws2uAXwNd/EYE24GuZ+w7OwhyYWblgGnAr51ziwKUKRDZ2gOdzOwhco+BFzOzw865/zmJ5UG2DGC5c26Tb5tPgHbAm0GQ7WZg0elSMLMvyP21nFeY2fxMBl69yH0LMxtmFgt8DNzjnAvkv2wLmq0tcKuZPQ9UAHLM7LhzLuAXKABhfzI2ityTqAn892RKozO2aUHuoY/EM5ZXIvf4WkXfYzNQKQhyFQNmAY8E26/ZGdsMJPAnYwvy6xbp2z7G93oCMDRIst0B/Nv3HtG+398bCzlbot/zG4EU3/NG/PRk7CYCezK2INkq+La/xcM/C3lmO2ObP3CJT8ZesjcuKg+gF7De9wfs175lI4Hevuf/JvfE13Lf41O/fe8l9+RTGrmHSDzPBdwNnPRbvhxoHgzZzniPgQS46APw+9kdWAGsBCYCxYIhG7l/Cb0GrAXWAC958Ov2MrknDZeTe76gkd++v/bttw7oGSzZyD1McuSMPwtVgyHbGe/xBy5x0WsKBBGREKeTsSIiIU5FLyIS4lT0IiIhTkUvIhLiVPQiIiFORS8iEuJU9CIiIe7/Ae0yJx3xdZWLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(step_means['exact'], step_means['correct'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6610fa1d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}