notebooks/nerf_table.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(datasets, experiment_name, metrics_list):\n",
    "    \"\"\"Given a list of datasets, compute the average metrics for each dataset.\"\"\"\n",
    "    metrics = [[] for _ in metrics_list]\n",
    "    for dataset in datasets:\n",
    "        filename = f\"../renders-postprocessed/{dataset}/{dataset}---{experiment_name}/{dataset}---{experiment_name}.json\"\n",
    "        with open(filename, \"r\") as f:\n",
    "            metrics_dict = json.load(f)\n",
    "            for idx, metric_name in enumerate(metrics_list):\n",
    "                metrics[idx].append(metrics_dict[metric_name])\n",
    "    # get the mean\n",
    "    metrics = [sum(metric) / len(metric) for metric in metrics]\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEMPLATE = \\\n",
    "\"\"\"\n",
    "\\\\begin{table*}[]\n",
    "    \\\\centering\n",
    "    \\\\begin{tabular}{l|lll|ll|lllll|l} % the number of columns needed\n",
    "        \\\\toprule\n",
    "        COLUMNS \\\\\\\\\n",
    "        \\\\midrule\n",
    "        EXPERIMENT_LINES\n",
    "        \\\\bottomrule\n",
    "    \\\\end{tabular}\n",
    "    \\\\caption{\\\\textbf{CAPTION_TITLE.} CAPTION_TEXT}\n",
    "    \\\\label{tab:TABLE_LABEL}\n",
    "\\\\end{table*}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "COLUMNS = \"\\n & PSNR $\\\\uparrow$ & SSIM $\\\\uparrow$ & LPIPS $\\\\downarrow$ & Depth $\\\\downarrow$ & Disp. $\\\\downarrow$ & Mean $^{\\\\circ}$ $\\\\downarrow$ & Median $^{\\\\circ}$ $\\\\downarrow$ & \\\\% $30^{\\\\circ}$ $\\\\uparrow$ & Coverage $\\\\uparrow$\"\n",
    "METRICS_LIST = [\"psnr\", \"ssim\", \"lpips\", \"depth\", \"disparity\", \"normals\", \"normals_median\", \"normals_30\", \"coverage\"]\n",
    "SIGFIGS = [2, 4, 4, 2, 2, 2, 2, 2, 2]\n",
    "assert len(METRICS_LIST) == len(SIGFIGS)\n",
    "# methods are the columns\n",
    "# here we map the title in the column to the the method name (which we can grab from folder names)\n",
    "# METHODS = [\n",
    "#     (\"Nerfacto Pseudo GT   \", \"nerfacto---pseudo-gt\"),\n",
    "#     (\"Nerfacto             \", \"nerfacto\"),\n",
    "#     (\" + Visibility Loss   \", \"nerfacto-visibility\"),\n",
    "#     (\" + Vis + Sparsity    \", \"nerfacto-visibility-sparsity\"),\n",
    "#     (\" + Vis + TV          \", \"nerfacto-visibility-TV\"),\n",
    "#     (\" + Vis + RegNeRF     \", \"nerfacto-visibility-regnerf\"),\n",
    "#     (\" + Vis + DSDS (Ours) \", \"nerfacto-visibility-cube\"),\n",
    "# ]\n",
    "# uncomment this for ablations\n",
    "METHODS = [\n",
    "    (\"Nerfacto GT\", \"nerfacto---pseudo-gt\"),\n",
    "    (\"Nerfacto          \", \"nerfacto\"),\n",
    "    (\" + Visibility Loss\", \"nerfacto-visibility\"),\n",
    "    (\" + Vis + RegNeRF  \", \"nerfacto-visibility-regnerf\"),\n",
    "    (\" + Vis + Sparsity \", \"nerfacto-visibility-sparsity\"),\n",
    "    (\" + Vis + TV       \", \"nerfacto-visibility-TV\"),\n",
    "    (\" + Vis + Cube     \", \"nerfacto-visibility-cube\"),\n",
    "    # sampling\n",
    "    (\" + Vis + Cube (sampling-densities)    \", \"nerfacto-frustum-cube-sampling-densities\"),\n",
    "    (\" + Vis + Cube (sampling-uniform)     \", \"nerfacto-frustum-cube-sampling-uniform\"),\n",
    "    # activation\n",
    "    (\" + Vis + Cube (activation-sigmoid)     \", \"nerfacto-frustum-cube-activation-sigmoid\"),\n",
    "    (\" + Vis + Cube (activation-clamp)     \", \"nerfacto-frustum-cube-activation-clamp\"),\n",
    "    # cubesize\n",
    "    (\" + Vis + Cube (cubescale-10-20)     \", \"nerfacto-frustum-cube-cubescale-10-20\"),\n",
    "    (\" + Vis + Cube (cubescale-01-20)    \", \"nerfacto-frustum-cube-cubescale-01-20\"),\n",
    "]\n",
    "DATASETS = [\n",
    "    \"aloe\",\n",
    "    \"art\",\n",
    "    \"car\",\n",
    "    \"century\",\n",
    "    \"flowers\",\n",
    "    \"garbage\",\n",
    "    \"picnic\",\n",
    "    \"pikachu\",\n",
    "    \"pipe\",\n",
    "    \"plant\",\n",
    "    \"roses\",\n",
    "    \"table\",\n",
    "]\n",
    "# DATASETS = ['aloe']\n",
    "# uncomment this for ablations\n",
    "DATASETS = ['garbage']\n",
    "\n",
    "def print_datasets(datasets, caption_title=\"caption title\", TABLE_LABEL=\"label\", caption_text=\"caption text\"):\n",
    "    metrics = get_metrics(datasets, \"nerfacto-visibility-sparsity\", METRICS_LIST)\n",
    "    metrics = [str(f\"{metric:.{sigfigs}f}\") for metric, sigfigs in zip(metrics, SIGFIGS)]\n",
    "\n",
    "    # create the experiment lines\n",
    "    EXPERIMENT_LINES = [\"\\n\"]\n",
    "    for idx, (method_name, experiment_name) in enumerate(METHODS):\n",
    "        if idx == 1:\n",
    "            line_string = f\"\\\\midrule \\n\"\n",
    "            EXPERIMENT_LINES.append(line_string)\n",
    "        metrics = get_metrics(datasets, experiment_name, METRICS_LIST)\n",
    "        metrics = [str(f\"{metric:.{sigfigs}f}\") for metric, sigfigs in zip(metrics, SIGFIGS)]\n",
    "        line_string = f\"{method_name} & {' & '.join(metrics)} \\\\\\\\ \\n\"\n",
    "        EXPERIMENT_LINES.append(line_string)\n",
    "\n",
    "    s = TEMPLATE.replace(\"COLUMNS\", COLUMNS)\n",
    "    s = s.replace(\"EXPERIMENT_LINES\", \" \".join(EXPERIMENT_LINES))\n",
    "    s = s.replace(\"CAPTION_TITLE\", caption_title)\n",
    "    s = s.replace(\"CAPTION_TEXT\", caption_text)\n",
    "    s = s.replace(\"TABLE_LABEL\", TABLE_LABEL)\n",
    "    print(s)\n",
    "\n",
    "# for the average of all the datasets\n",
    "print_datasets(DATASETS, caption_title=\"``Average\\\" capture quantitative evaluation\", TABLE_LABEL=\"average\")\n",
    "\n",
    "print(\"---\" * 10)\n",
    "\n",
    "# for all the datasets individually\n",
    "for i in range(len(DATASETS)):\n",
    "    caption_title = f\"\"\"``{DATASETS[i]}\\\"\"\"\" + \" capture quantitative evaluation\"\n",
    "    caption_text = \"\"\"Per scene results that compare Nerfacto with baselines (that uses geometric handcrafted regularizers) and Nerfbusters (that uses a data-driven local 3D prior). \"\"\" + \\\n",
    "        \"\"\"We highlight that all baselines use the proposed visibility loss. \\\\cref{sec:evaluation_procedure} describes metrics in more detail, $\\\\uparrow$/$\\\\downarrow$ indicates if higher/lower is better.\"\"\"\n",
    "    print_datasets(DATASETS[i : i + 1], caption_title=caption_title, TABLE_LABEL=DATASETS[i], caption_text=caption_text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nerfstudio",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}