From 75ea6e75d0aebf14cb92747ece298b5d502a6873 Mon Sep 17 00:00:00 2001
From: Jett <jettjaniak@gmail.com>
Date: Wed, 24 Apr 2024 10:58:17 -0400
Subject: [PATCH] small update

---
 notebooks/eval_notebook.ipynb | 87 ++++++++++++++++++++---------------
 src/delphi/eval/vis.py        |  2 -
 2 files changed, 51 insertions(+), 38 deletions(-)

diff --git a/notebooks/eval_notebook.ipynb b/notebooks/eval_notebook.ipynb
index 88742ec5..79bc02fd 100644
--- a/notebooks/eval_notebook.ipynb
+++ b/notebooks/eval_notebook.ipynb
@@ -21,12 +21,12 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "data": {
-      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.4.0'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var reloading = false;\n  var Bokeh = root.Bokeh;\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n      root._bokeh_is_loading = css_urls.length + 0;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.4.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.4.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.4.0.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.4.0.min.js\", \"https://cdn.holoviz.org/panel/1.4.0/dist/panel.min.js\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n\ttry {\n          inline_js[i].call(root, root.Bokeh);\n\t} catch(e) {\n\t  if (!reloading) {\n\t    throw e;\n\t  }\n\t}\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      if (!reloading && !bokeh_loaded) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
+      "application/javascript": "(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n  var py_version = '3.4.1'.replace('rc', '-rc.').replace('.dev', '-dev.');\n  var reloading = false;\n  var Bokeh = root.Bokeh;\n\n  if (typeof (root._bokeh_timeout) === \"undefined\" || force) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) {\n        if (callback != null)\n          callback();\n      });\n    } finally {\n      delete root._bokeh_onload_callbacks;\n    }\n    console.debug(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(css_urls, js_urls, js_modules, js_exports, callback) {\n    if (css_urls == null) css_urls = [];\n    if (js_urls == null) js_urls = [];\n    if (js_modules == null) js_modules = [];\n    if (js_exports == null) js_exports = {};\n\n    root._bokeh_onload_callbacks.push(callback);\n\n    if (root._bokeh_is_loading > 0) {\n      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls.length === 0 && js_modules.length === 0 && Object.keys(js_exports).length === 0) {\n      run_callbacks();\n      return null;\n    }\n    if (!reloading) {\n      console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    }\n\n    function on_load() {\n      root._bokeh_is_loading--;\n      if (root._bokeh_is_loading === 0) {\n        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n        run_callbacks()\n      }\n    }\n    window._bokeh_on_load = on_load\n\n    function on_error() {\n      console.error(\"failed to load \" + url);\n    }\n\n    var skip = [];\n    if (window.requirejs) {\n      window.requirejs.config({'packages': {}, 'paths': {}, 'shim': {}});\n      root._bokeh_is_loading = css_urls.length + 0;\n    } else {\n      root._bokeh_is_loading = css_urls.length + js_urls.length + js_modules.length + Object.keys(js_exports).length;\n    }\n\n    var existing_stylesheets = []\n    var links = document.getElementsByTagName('link')\n    for (var i = 0; i < links.length; i++) {\n      var link = links[i]\n      if (link.href != null) {\n\texisting_stylesheets.push(link.href)\n      }\n    }\n    for (var i = 0; i < css_urls.length; i++) {\n      var url = css_urls[i];\n      if (existing_stylesheets.indexOf(url) !== -1) {\n\ton_load()\n\tcontinue;\n      }\n      const element = document.createElement(\"link\");\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.rel = \"stylesheet\";\n      element.type = \"text/css\";\n      element.href = url;\n      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n      document.body.appendChild(element);\n    }    var existing_scripts = []\n    var scripts = document.getElementsByTagName('script')\n    for (var i = 0; i < scripts.length; i++) {\n      var script = scripts[i]\n      if (script.src != null) {\n\texisting_scripts.push(script.src)\n      }\n    }\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (var i = 0; i < js_modules.length; i++) {\n      var url = js_modules[i];\n      if (skip.indexOf(url) !== -1 || existing_scripts.indexOf(url) !== -1) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onload = on_load;\n      element.onerror = on_error;\n      element.async = false;\n      element.src = url;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.head.appendChild(element);\n    }\n    for (const name in js_exports) {\n      var url = js_exports[name];\n      if (skip.indexOf(url) >= 0 || root[name] != null) {\n\tif (!window.requirejs) {\n\t  on_load();\n\t}\n\tcontinue;\n      }\n      var element = document.createElement('script');\n      element.onerror = on_error;\n      element.async = false;\n      element.type = \"module\";\n      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      element.textContent = `\n      import ${name} from \"${url}\"\n      window.${name} = ${name}\n      window._bokeh_on_load()\n      `\n      document.head.appendChild(element);\n    }\n    if (!js_urls.length && !js_modules.length) {\n      on_load()\n    }\n  };\n\n  function inject_raw_css(css) {\n    const element = document.createElement(\"style\");\n    element.appendChild(document.createTextNode(css));\n    document.body.appendChild(element);\n  }\n\n  var js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.4.1.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.4.1.min.js\", \"https://cdn.holoviz.org/panel/1.4.1/dist/panel.min.js\"];\n  var js_modules = [];\n  var js_exports = {};\n  var css_urls = [];\n  var inline_js = [    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\nfunction(Bokeh) {} // ensure no trailing comma for IE\n  ];\n\n  function run_inline_js() {\n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n\ttry {\n          inline_js[i].call(root, root.Bokeh);\n\t} catch(e) {\n\t  if (!reloading) {\n\t    throw e;\n\t  }\n\t}\n      }\n      // Cache old bokeh versions\n      if (Bokeh != undefined && !reloading) {\n\tvar NewBokeh = root.Bokeh;\n\tif (Bokeh.versions === undefined) {\n\t  Bokeh.versions = new Map();\n\t}\n\tif (NewBokeh.version !== Bokeh.version) {\n\t  Bokeh.versions.set(NewBokeh.version, NewBokeh)\n\t}\n\troot.Bokeh = Bokeh;\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    }\n    root._bokeh_is_initializing = false\n  }\n\n  function load_or_wait() {\n    // Implement a backoff loop that tries to ensure we do not load multiple\n    // versions of Bokeh and its dependencies at the same time.\n    // In recent versions we use the root._bokeh_is_initializing flag\n    // to determine whether there is an ongoing attempt to initialize\n    // bokeh, however for backward compatibility we also try to ensure\n    // that we do not start loading a newer (Panel>=1.0 and Bokeh>3) version\n    // before older versions are fully initialized.\n    if (root._bokeh_is_initializing && Date.now() > root._bokeh_timeout) {\n      root._bokeh_is_initializing = false;\n      root._bokeh_onload_callbacks = undefined;\n      console.log(\"Bokeh: BokehJS was loaded multiple times but one version failed to initialize.\");\n      load_or_wait();\n    } else if (root._bokeh_is_initializing || (typeof root._bokeh_is_initializing === \"undefined\" && root._bokeh_onload_callbacks !== undefined)) {\n      setTimeout(load_or_wait, 100);\n    } else {\n      root._bokeh_is_initializing = true\n      root._bokeh_onload_callbacks = []\n      var bokeh_loaded = Bokeh != null && (Bokeh.version === py_version || (Bokeh.versions !== undefined && Bokeh.versions.has(py_version)));\n      if (!reloading && !bokeh_loaded) {\n\troot.Bokeh = undefined;\n      }\n      load_libs(css_urls, js_urls, js_modules, js_exports, function() {\n\tconsole.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n\trun_inline_js();\n      });\n    }\n  }\n  // Give older versions of the autoload script a head-start to ensure\n  // they initialize before we start loading newer version.\n  setTimeout(load_or_wait, 100)\n}(window));",
       "application/vnd.holoviews_load.v0+json": ""
      },
      "metadata": {},
@@ -68,12 +68,12 @@
      "data": {
       "application/vnd.holoviews_exec.v0+json": "",
       "text/html": [
-       "<div id='7f14ad27-be2c-4041-9b01-3e14b9fc8d40'>\n",
-       "  <div id=\"d3697b61-4de9-4bc7-821d-05b86c4fb8f7\" data-root-id=\"7f14ad27-be2c-4041-9b01-3e14b9fc8d40\" style=\"display: contents;\"></div>\n",
+       "<div id='3f76b517-d472-4be5-a5e7-914bf5e2be7d'>\n",
+       "  <div id=\"e7820ec9-5324-4997-8313-6c9774940d0c\" data-root-id=\"3f76b517-d472-4be5-a5e7-914bf5e2be7d\" style=\"display: contents;\"></div>\n",
        "</div>\n",
        "<script type=\"application/javascript\">(function(root) {\n",
-       "  var docs_json = {\"978ed142-227d-4d43-b625-ff3992ba942e\":{\"version\":\"3.4.0\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"panel.models.browser.BrowserInfo\",\"id\":\"7f14ad27-be2c-4041-9b01-3e14b9fc8d40\"},{\"type\":\"object\",\"name\":\"panel.models.comm_manager.CommManager\",\"id\":\"2c8d2026-d94d-483c-b550-9ae8d1a9c7af\",\"attributes\":{\"plot_id\":\"7f14ad27-be2c-4041-9b01-3e14b9fc8d40\",\"comm_id\":\"2bb31bf0656242819d8d3f9a190e217f\",\"client_comm_id\":\"3cc8c97f46eb43fc87cc62442d4c2395\"}}],\"defs\":[{\"type\":\"model\",\"name\":\"ReactiveHTML1\"},{\"type\":\"model\",\"name\":\"FlexBox1\",\"properties\":[{\"name\":\"align_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"align_items\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"flex_direction\",\"kind\":\"Any\",\"default\":\"row\"},{\"name\":\"flex_wrap\",\"kind\":\"Any\",\"default\":\"wrap\"},{\"name\":\"justify_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"gap\",\"kind\":\"Any\",\"default\":\"\"}]},{\"type\":\"model\",\"name\":\"FloatPanel1\",\"properties\":[{\"name\":\"config\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"contained\",\"kind\":\"Any\",\"default\":true},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"right-top\"},{\"name\":\"offsetx\",\"kind\":\"Any\",\"default\":null},{\"name\":\"offsety\",\"kind\":\"Any\",\"default\":null},{\"name\":\"theme\",\"kind\":\"Any\",\"default\":\"primary\"},{\"name\":\"status\",\"kind\":\"Any\",\"default\":\"normalized\"}]},{\"type\":\"model\",\"name\":\"GridStack1\",\"properties\":[{\"name\":\"mode\",\"kind\":\"Any\",\"default\":\"warn\"},{\"name\":\"ncols\",\"kind\":\"Any\",\"default\":null},{\"name\":\"nrows\",\"kind\":\"Any\",\"default\":null},{\"name\":\"allow_resize\",\"kind\":\"Any\",\"default\":true},{\"name\":\"allow_drag\",\"kind\":\"Any\",\"default\":true},{\"name\":\"state\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"drag1\",\"properties\":[{\"name\":\"slider_width\",\"kind\":\"Any\",\"default\":5},{\"name\":\"slider_color\",\"kind\":\"Any\",\"default\":\"black\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":50}]},{\"type\":\"model\",\"name\":\"click1\",\"properties\":[{\"name\":\"terminal_output\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"debug_name\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"clears\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"FastWrapper1\",\"properties\":[{\"name\":\"object\",\"kind\":\"Any\",\"default\":null},{\"name\":\"style\",\"kind\":\"Any\",\"default\":null}]},{\"type\":\"model\",\"name\":\"NotificationAreaBase1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"NotificationArea1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"notifications\",\"kind\":\"Any\",\"default\":[]},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0},{\"name\":\"types\",\"kind\":\"Any\",\"default\":[{\"type\":\"map\",\"entries\":[[\"type\",\"warning\"],[\"background\",\"#ffc107\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-exclamation-triangle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]},{\"type\":\"map\",\"entries\":[[\"type\",\"info\"],[\"background\",\"#007bff\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-info-circle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]}]}]},{\"type\":\"model\",\"name\":\"Notification\",\"properties\":[{\"name\":\"background\",\"kind\":\"Any\",\"default\":null},{\"name\":\"duration\",\"kind\":\"Any\",\"default\":3000},{\"name\":\"icon\",\"kind\":\"Any\",\"default\":null},{\"name\":\"message\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"notification_type\",\"kind\":\"Any\",\"default\":null},{\"name\":\"_destroyed\",\"kind\":\"Any\",\"default\":false}]},{\"type\":\"model\",\"name\":\"TemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"BootstrapTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"TemplateEditor1\",\"properties\":[{\"name\":\"layout\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"MaterialTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"copy_to_clipboard1\",\"properties\":[{\"name\":\"fill\",\"kind\":\"Any\",\"default\":\"none\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":null}]}]}};\n",
-       "  var render_items = [{\"docid\":\"978ed142-227d-4d43-b625-ff3992ba942e\",\"roots\":{\"7f14ad27-be2c-4041-9b01-3e14b9fc8d40\":\"d3697b61-4de9-4bc7-821d-05b86c4fb8f7\"},\"root_ids\":[\"7f14ad27-be2c-4041-9b01-3e14b9fc8d40\"]}];\n",
+       "  var docs_json = {\"2c6a4952-472e-4bd0-8584-c3619ac4d449\":{\"version\":\"3.4.1\",\"title\":\"Bokeh Application\",\"roots\":[{\"type\":\"object\",\"name\":\"panel.models.browser.BrowserInfo\",\"id\":\"3f76b517-d472-4be5-a5e7-914bf5e2be7d\"},{\"type\":\"object\",\"name\":\"panel.models.comm_manager.CommManager\",\"id\":\"e60441ea-d812-41b7-954b-c8f37ffbae5a\",\"attributes\":{\"plot_id\":\"3f76b517-d472-4be5-a5e7-914bf5e2be7d\",\"comm_id\":\"02fdf95a29b741da86ee380f452ea627\",\"client_comm_id\":\"9917e3cbc9764e86b2c6763d09a494c4\"}}],\"defs\":[{\"type\":\"model\",\"name\":\"ReactiveHTML1\"},{\"type\":\"model\",\"name\":\"FlexBox1\",\"properties\":[{\"name\":\"align_content\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"align_items\",\"kind\":\"Any\",\"default\":\"flex-start\"},{\"name\":\"flex_direction\",\"kind\":\"Any\",\"default\":\"row\"},{\"name\":\"flex_wrap\",\"kind\":\"Any\",\"default\":\"wrap\"},{\"name\":\"gap\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"justify_content\",\"kind\":\"Any\",\"default\":\"flex-start\"}]},{\"type\":\"model\",\"name\":\"FloatPanel1\",\"properties\":[{\"name\":\"config\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"contained\",\"kind\":\"Any\",\"default\":true},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"right-top\"},{\"name\":\"offsetx\",\"kind\":\"Any\",\"default\":null},{\"name\":\"offsety\",\"kind\":\"Any\",\"default\":null},{\"name\":\"theme\",\"kind\":\"Any\",\"default\":\"primary\"},{\"name\":\"status\",\"kind\":\"Any\",\"default\":\"normalized\"}]},{\"type\":\"model\",\"name\":\"GridStack1\",\"properties\":[{\"name\":\"mode\",\"kind\":\"Any\",\"default\":\"warn\"},{\"name\":\"ncols\",\"kind\":\"Any\",\"default\":null},{\"name\":\"nrows\",\"kind\":\"Any\",\"default\":null},{\"name\":\"allow_resize\",\"kind\":\"Any\",\"default\":true},{\"name\":\"allow_drag\",\"kind\":\"Any\",\"default\":true},{\"name\":\"state\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"drag1\",\"properties\":[{\"name\":\"slider_width\",\"kind\":\"Any\",\"default\":5},{\"name\":\"slider_color\",\"kind\":\"Any\",\"default\":\"black\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":50}]},{\"type\":\"model\",\"name\":\"click1\",\"properties\":[{\"name\":\"terminal_output\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"debug_name\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"clears\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"FastWrapper1\",\"properties\":[{\"name\":\"object\",\"kind\":\"Any\",\"default\":null},{\"name\":\"style\",\"kind\":\"Any\",\"default\":null}]},{\"type\":\"model\",\"name\":\"NotificationAreaBase1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"NotificationArea1\",\"properties\":[{\"name\":\"js_events\",\"kind\":\"Any\",\"default\":{\"type\":\"map\"}},{\"name\":\"notifications\",\"kind\":\"Any\",\"default\":[]},{\"name\":\"position\",\"kind\":\"Any\",\"default\":\"bottom-right\"},{\"name\":\"_clear\",\"kind\":\"Any\",\"default\":0},{\"name\":\"types\",\"kind\":\"Any\",\"default\":[{\"type\":\"map\",\"entries\":[[\"type\",\"warning\"],[\"background\",\"#ffc107\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-exclamation-triangle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]},{\"type\":\"map\",\"entries\":[[\"type\",\"info\"],[\"background\",\"#007bff\"],[\"icon\",{\"type\":\"map\",\"entries\":[[\"className\",\"fas fa-info-circle\"],[\"tagName\",\"i\"],[\"color\",\"white\"]]}]]}]}]},{\"type\":\"model\",\"name\":\"Notification\",\"properties\":[{\"name\":\"background\",\"kind\":\"Any\",\"default\":null},{\"name\":\"duration\",\"kind\":\"Any\",\"default\":3000},{\"name\":\"icon\",\"kind\":\"Any\",\"default\":null},{\"name\":\"message\",\"kind\":\"Any\",\"default\":\"\"},{\"name\":\"notification_type\",\"kind\":\"Any\",\"default\":null},{\"name\":\"_destroyed\",\"kind\":\"Any\",\"default\":false}]},{\"type\":\"model\",\"name\":\"TemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"BootstrapTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"TemplateEditor1\",\"properties\":[{\"name\":\"layout\",\"kind\":\"Any\",\"default\":[]}]},{\"type\":\"model\",\"name\":\"MaterialTemplateActions1\",\"properties\":[{\"name\":\"open_modal\",\"kind\":\"Any\",\"default\":0},{\"name\":\"close_modal\",\"kind\":\"Any\",\"default\":0}]},{\"type\":\"model\",\"name\":\"copy_to_clipboard1\",\"properties\":[{\"name\":\"fill\",\"kind\":\"Any\",\"default\":\"none\"},{\"name\":\"value\",\"kind\":\"Any\",\"default\":null}]}]}};\n",
+       "  var render_items = [{\"docid\":\"2c6a4952-472e-4bd0-8584-c3619ac4d449\",\"roots\":{\"3f76b517-d472-4be5-a5e7-914bf5e2be7d\":\"e7820ec9-5324-4997-8313-6c9774940d0c\"},\"root_ids\":[\"3f76b517-d472-4be5-a5e7-914bf5e2be7d\"]}];\n",
        "  var docs = Object.values(docs_json)\n",
        "  if (!docs) {\n",
        "    return\n",
@@ -137,7 +137,7 @@
      },
      "metadata": {
       "application/vnd.holoviews_exec.v0+json": {
-       "id": "7f14ad27-be2c-4041-9b01-3e14b9fc8d40"
+       "id": "3f76b517-d472-4be5-a5e7-914bf5e2be7d"
       }
      },
      "output_type": "display_data"
@@ -165,7 +165,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -180,7 +180,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -195,13 +195,21 @@
     "    .map(lambda x: {\"logprobs\": x[\"logprobs\"].to(device)})\n",
     "    for suffix in suffixes\n",
     "}\n",
-    "next_logprobs_plot = {k: d[\"logprobs\"] for k, d in next_logprobs.items()}\n",
+    "next_logprobs_plot = {k: d[\"logprobs\"] for k, d in next_logprobs.items()}\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "\n",
     "# load the tokenized dataset\n",
     "tokenized_corpus_dataset = (\n",
     "    cast(\n",
     "        Dataset,\n",
-    "        load_dataset(\"delphi-suite/tinystories-v2-clean-tokenized-v0\", split=split),\n",
+    "        load_dataset(\"delphi-suite/stories-tokenized\", split=split),\n",
     "    )\n",
     "    .with_format(\"torch\")\n",
     "    .map(lambda x: {\"tokens\": x[\"tokens\"].to(device)})\n",
@@ -217,21 +225,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d9729e75a91e4ed5872b2532979ef9f4",
+       "model_id": "e129e76af7f944188217d1fd1a05a533",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "BokehModel(combine_events=True, render_bundle={'docs_json': {'44dc30e4-17f8-472e-8dff-d0a221af9c6b': {'version…"
+       "BokehModel(combine_events=True, render_bundle={'docs_json': {'fa6054c6-b791-4936-995f-720a7b18e37a': {'version…"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 17,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -239,12 +247,16 @@
    "source": [
     "# specific token specification\n",
     "tokenizer = AutoTokenizer.from_pretrained(\"delphi-suite/stories-tokenizer\")\n",
-    "# get only the tokens present in the dataset\n",
-    "valid_tok_ids = torch.unique(\n",
-    "    tokenized_corpus_dataset[\"tokens\"]\n",
-    ").tolist()  # use torch.unique instead of np.unique when tokenized_corpus_dataset is a torch tensor\n",
-    "valid_toks = tokenizer.convert_ids_to_tokens(valid_tok_ids)\n",
-    "vocab = {tok: valid_tok_ids[i] for i, tok in enumerate(valid_toks)}\n",
+    "\n",
+    "# Count the frequency of each token using torch.bincount\n",
+    "token_counts = torch.bincount(tokenized_corpus_dataset[\"tokens\"].view(-1))\n",
+    "\n",
+    "# Get the indices that would sort the token counts in descending order\n",
+    "sorted_indices = torch.argsort(token_counts, descending=True)\n",
+    "\n",
+    "# Get the token IDs in descending order of frequency\n",
+    "valid_tok_ids = sorted_indices.tolist()\n",
+    "vocab = {tokenizer.decode(t): t for t in sorted_indices.tolist() if token_counts[t] > 0}\n",
     "\n",
     "selector, selected_ids = token_selector(vocab)  # use selected_ids as a dynamic variable\n",
     "pn.Column(selector, height=300).servable()"
@@ -252,24 +264,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Selected IDs: [260, 262, 261, 263, 264, 265, 267, 268]\n"
+      "Selected IDs: [40, 2, 14]\n"
      ]
     }
    ],
    "source": [
+    "if not selected_ids:\n",
+    "    selected_ids = [40, 2, 14]\n",
     "print(\"Selected IDs:\", selected_ids)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -284,7 +298,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "19c3348321234a3ea0891172f6e377bd",
+       "model_id": "6ff632bf47ca4f529eb2b469b7644548",
        "version_major": 2,
        "version_minor": 0
       },
@@ -296,9 +310,9 @@
        "              'name': 'Upper Bound',\n",
        "              'showlegend': False,\n",
        "              'type': 'scatter',\n",
-       "              'uid': '9eae4162-972e-4209-8aef-419d27ec28f1',\n",
+       "              'uid': 'c4f9fbcc-fa0e-4fc7-a64d-a36c7ca93e22',\n",
        "              'x': [100k, 200k, 400k],\n",
-       "              'y': array([1.51873401, 1.37054035, 1.20272651])},\n",
+       "              'y': array([5.13345224, 4.86881149, 4.3468231 ])},\n",
        "             {'fill': 'tonexty',\n",
        "              'fillcolor': 'rgba(68, 68, 68, 0.3)',\n",
        "              'line': {'width': 0},\n",
@@ -307,21 +321,21 @@
        "              'name': 'Lower Bound',\n",
        "              'showlegend': False,\n",
        "              'type': 'scatter',\n",
-       "              'uid': '8bb5125d-0df0-4756-89e2-33624e31dee7',\n",
+       "              'uid': 'f1a2f5a9-ef40-4e26-8319-973ebd426a82',\n",
        "              'x': [100k, 200k, 400k],\n",
-       "              'y': array([0.30299246, 0.27227049, 0.23416051])},\n",
+       "              'y': array([1.06806397, 1.09191781, 0.88462193])},\n",
        "             {'marker': {'color': 'rgb(31, 119, 180)', 'line': {'color': 'rgb(31, 119, 180)', 'width': 1}, 'size': 0},\n",
        "              'mode': 'lines',\n",
        "              'name': 'Means',\n",
        "              'type': 'scatter',\n",
-       "              'uid': '137d05e8-bb5d-4dff-8b7c-2d47cfbefdd9',\n",
+       "              'uid': '1944ef31-e955-426c-b8d0-4c5744e0192d',\n",
        "              'x': [100k, 200k, 400k],\n",
-       "              'y': array([0.4441171 , 0.37617055, 0.29508319])}],\n",
+       "              'y': array([1.83951139, 1.53821385, 1.22834492])}],\n",
        "    'layout': {'template': '...'}\n",
        "})"
       ]
      },
-     "execution_count": 9,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -344,7 +358,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -353,13 +367,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "761e1bdeadb348f18b827f5336e6c406",
+       "model_id": "991c21fe8cbe4d909320aa86a2ca1d5c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -403,7 +417,8 @@
     "        description=\"Model 2\",\n",
     "        value=\"200k\",\n",
     "    ),\n",
-    ")"
+    ")\n",
+    "# FIXME: why is the prompt showing twice?"
    ]
   },
   {
diff --git a/src/delphi/eval/vis.py b/src/delphi/eval/vis.py
index 737846e3..c26fb545 100644
--- a/src/delphi/eval/vis.py
+++ b/src/delphi/eval/vis.py
@@ -186,7 +186,6 @@ def vis_pos_map(
     metrics: Float[torch.Tensor, "prompt pos"],
     token_ids: Int[torch.Tensor, "prompt pos"],
     tokenizer: PreTrainedTokenizerBase,
-    display_html: bool = False,
 ):
     """
     Randomly sample from pos_map and visualize the loss diff at the corresponding position.
@@ -251,7 +250,6 @@ def vis_pos_map(
     </script>
     """
     display(HTML(html_str))
-    return html_str
 
 
 def token_selector(