new GT deletion endpoint

nucleus/dataset.py

@@ -1181,6 +1181,17 @@ def delete_annotations(
         )
         return AsyncJob.from_json(response, self._client)
 
+    def delete_annotation_refloc(
+        self,
+        reference_id: str,
+        annotation_id: str,
+    ):
+        return self._client.make_request(
+            {},
+            f"dataset/{self.id}/annotation/{reference_id}/{annotation_id}",
+            requests_command=requests.delete,
+        )
+
     def get_scene(self, reference_id: str) -> Scene:
         """Fetches a single scene in the dataset by its reference ID.
 

nucleus/metrics/cuboid_metrics.py

@@ -1,4 +1,5 @@
 import sys
+import warnings
 from abc import abstractmethod
 from typing import List, Optional, Union
 
@@ -10,6 +11,9 @@
 from .filtering import ListOfAndFilters, ListOfOrAndFilters
 from .filters import confidence_filter
 
+DEFAULT_IOU_THRESHOLD = 0.1
+DEFAULT_CONFIDENCE_THRESHOLD = 0.0
+
 
 class CuboidMetric(Metric):
     """Abstract class for metrics of cuboids.
@@ -27,8 +31,9 @@ class CuboidMetric(Metric):
 
     def __init__(
         self,
+        iou_threshold: float,
         enforce_label_match: bool = False,
-        confidence_threshold: float = 0.0,
+        confidence_threshold: Optional[float] = None,
         annotation_filters: Optional[
             Union[ListOfOrAndFilters, ListOfAndFilters]
         ] = None,
@@ -54,6 +59,11 @@ def __init__(
                 (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines
                 these filters as a disjunction (OR).
         """
+        if not confidence_threshold:
+            confidence_threshold = DEFAULT_CONFIDENCE_THRESHOLD
+            warnings.warn(
+                f"Got confidence_threshold value of `None`. In this case, we set the confidence_threshold to {confidence_threshold} (include all predictions, regardless of confidence).  Consider specifying this value explicitly during metric initialization"
+            )
         self.enforce_label_match = enforce_label_match
         assert 0 <= confidence_threshold <= 1
         self.confidence_threshold = confidence_threshold
@@ -99,8 +109,8 @@ class CuboidIOU(CuboidMetric):
     def __init__(
         self,
         enforce_label_match: bool = True,
-        iou_threshold: float = 0.0,
-        confidence_threshold: float = 0.0,
+        iou_threshold: Optional[float] = None,
+        confidence_threshold: Optional[float] = None,
         iou_2d: bool = False,
         annotation_filters: Optional[
             Union[ListOfOrAndFilters, ListOfAndFilters]
@@ -127,13 +137,19 @@ def __init__(
                 interpreted as a conjunction (AND), forming a more selective and multiple column predicate.
                 Finally, the most outer list combines these filters as a disjunction (OR).
         """
+        if not iou_threshold:
+            iou_threshold = DEFAULT_IOU_THRESHOLD
+            warnings.warn(
+                f"The IoU threshold used for matching was initialized to `None`. In this case, the value of iou_threshold defaults to {iou_threshold}. If this values will produce unexpected behavior, consider specifying the iou_threshold argument during metric initialization"
+            )
         assert (
             0 <= iou_threshold <= 1
         ), "IoU threshold must be between 0 and 1."
         self.iou_threshold = iou_threshold
         self.iou_2d = iou_2d
         super().__init__(
             enforce_label_match=enforce_label_match,
+            iou_threshold=iou_threshold,
             confidence_threshold=confidence_threshold,
             annotation_filters=annotation_filters,
             prediction_filters=prediction_filters,
@@ -147,14 +163,16 @@ def eval(
         iou_3d_metric, iou_2d_metric = detection_iou(
             predictions,
             annotations,
-            threshold_in_overlap_ratio=self.iou_threshold,
+            self.iou_threshold,
         )
 
-        weight = max(len(annotations), len(predictions))
+        # If there are zero IoU matches, avg_iou defaults to value 0
         if self.iou_2d:
-            avg_iou = iou_2d_metric.sum() / max(weight, sys.float_info.epsilon)
+            weight = len(iou_2d_metric)
+            avg_iou = iou_2d_metric.sum() / weight if weight > 0 else 0.0
         else:
-            avg_iou = iou_3d_metric.sum() / max(weight, sys.float_info.epsilon)
+            weight = len(iou_3d_metric)
+            avg_iou = iou_3d_metric.sum() / weight if weight > 0 else 0.0
 
         return ScalarResult(avg_iou, weight)
 
@@ -166,8 +184,8 @@ class CuboidPrecision(CuboidMetric):
     def __init__(
         self,
         enforce_label_match: bool = True,
-        iou_threshold: float = 0.0,
-        confidence_threshold: float = 0.0,
+        iou_threshold: Optional[float] = None,
+        confidence_threshold: Optional[float] = None,
         annotation_filters: Optional[
             Union[ListOfOrAndFilters, ListOfAndFilters]
         ] = None,
@@ -192,12 +210,18 @@ def __init__(
                 interpreted as a conjunction (AND), forming a more selective and multiple column predicate.
                 Finally, the most outer list combines these filters as a disjunction (OR).
         """
+        if not iou_threshold:
+            iou_threshold = DEFAULT_IOU_THRESHOLD
+            warnings.warn(
+                f"The IoU threshold used for matching was initialized to `None`. In this case, the value of iou_threshold defaults to {iou_threshold}. If this values will produce unexpected behavior, consider specifying the iou_threshold argument during metric initialization"
+            )
         assert (
             0 <= iou_threshold <= 1
         ), "IoU threshold must be between 0 and 1."
         self.iou_threshold = iou_threshold
         super().__init__(
             enforce_label_match=enforce_label_match,
+            iou_threshold=iou_threshold,
             confidence_threshold=confidence_threshold,
             annotation_filters=annotation_filters,
             prediction_filters=prediction_filters,
@@ -211,7 +235,7 @@ def eval(
         stats = recall_precision(
             predictions,
             annotations,
-            threshold_in_overlap_ratio=self.iou_threshold,
+            self.iou_threshold,
         )
         weight = stats["tp_sum"] + stats["fp_sum"]
         precision = stats["tp_sum"] / max(weight, sys.float_info.epsilon)
@@ -225,8 +249,8 @@ class CuboidRecall(CuboidMetric):
     def __init__(
         self,
         enforce_label_match: bool = True,
-        iou_threshold: float = 0.0,
-        confidence_threshold: float = 0.0,
+        iou_threshold: Optional[float] = None,
+        confidence_threshold: Optional[float] = None,
         annotation_filters: Optional[
             Union[ListOfOrAndFilters, ListOfAndFilters]
         ] = None,
@@ -241,12 +265,18 @@ def __init__(
             iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0
             confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0
         """
+        if not iou_threshold:
+            iou_threshold = DEFAULT_IOU_THRESHOLD
+            warnings.warn(
+                f"The IoU threshold used for matching was initialized to `None`. In this case, the value of iou_threshold defaults to {iou_threshold}. If this values will produce unexpected behavior, consider specifying the iou_threshold argument during metric initialization"
+            )
         assert (
             0 <= iou_threshold <= 1
         ), "IoU threshold must be between 0 and 1."
         self.iou_threshold = iou_threshold
         super().__init__(
             enforce_label_match=enforce_label_match,
+            iou_threshold=iou_threshold,
             confidence_threshold=confidence_threshold,
             annotation_filters=annotation_filters,
             prediction_filters=prediction_filters,
@@ -260,7 +290,7 @@ def eval(
         stats = recall_precision(
             predictions,
             annotations,
-            threshold_in_overlap_ratio=self.iou_threshold,
+            self.iou_threshold,
         )
         weight = stats["tp_sum"] + stats["fn_sum"]
         recall = stats["tp_sum"] / max(weight, sys.float_info.epsilon)

nucleus/metrics/cuboid_utils.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from functools import wraps
 from typing import Dict, List, Tuple
 
@@ -35,6 +36,14 @@ def __init__(self, *args, **kwargs):
 from .base import ScalarResult
 
 
+@dataclass
+class ProcessedCuboids:
+    xyz: np.array
+    wlh: np.array
+    yaw: np.array
+    labels: List[str]
+
+
 def group_cuboids_by_label(
     annotations: List[CuboidAnnotation],
     predictions: List[CuboidPrediction],
@@ -101,19 +110,19 @@ def wrapper(
     return wrapper
 
 
-def process_dataitem(dataitem):
-    processed_item = {}
-    processed_item["xyz"] = np.array(
-        [[ann.position.x, ann.position.y, ann.position.z] for ann in dataitem]
+def process_cuboids(item_list):
+    xyz = np.array(
+        [[ann.position.x, ann.position.y, ann.position.z] for ann in item_list]
     )
-    processed_item["wlh"] = np.array(
+    wlh = np.array(
         [
             [ann.dimensions.x, ann.dimensions.y, ann.dimensions.z]
-            for ann in dataitem
+            for ann in item_list
         ]
     )
-    processed_item["yaw"] = np.array([ann.yaw for ann in dataitem])
-    return processed_item
+    yaw = np.array([ann.yaw for ann in item_list])
+    labels = [ann.label for ann in item_list]
+    return ProcessedCuboids(xyz, wlh, yaw, labels)
 
 
 def compute_outer_iou(
@@ -178,7 +187,6 @@ def compute_outer_iou(
                     .intersection(polygon_1)
                     .area
                 )
-
     intersection = height_intersection * area_intersection
     area_0 = wlh_0[:, 0] * wlh_0[:, 1]
     area_1 = wlh_1[:, 0] * wlh_1[:, 1]
@@ -294,23 +302,23 @@ def recall_precision(
     num_predicted = 0
     num_instances = 0
 
-    gt_items = process_dataitem(groundtruth)
-    pred_items = process_dataitem(prediction)
+    gt_items = process_cuboids(groundtruth)
+    pred_items = process_cuboids(prediction)
 
-    num_predicted += pred_items["xyz"].shape[0]
-    num_instances += gt_items["xyz"].shape[0]
+    num_predicted += pred_items.xyz.shape[0]
+    num_instances += gt_items.xyz.shape[0]
 
-    tp = np.zeros(pred_items["xyz"].shape[0])
-    fp = np.ones(pred_items["xyz"].shape[0])
-    fn = np.ones(gt_items["xyz"].shape[0])
+    tp = np.zeros(pred_items.xyz.shape[0])
+    fp = np.ones(pred_items.xyz.shape[0])
+    fn = np.ones(gt_items.xyz.shape[0])
 
     mapping = associate_cuboids_on_iou(
-        pred_items["xyz"],
-        pred_items["wlh"],
-        pred_items["yaw"] + np.pi / 2,
-        gt_items["xyz"],
-        gt_items["wlh"],
-        gt_items["yaw"] + np.pi / 2,
+        pred_items.xyz,
+        pred_items.wlh,
+        pred_items.yaw + np.pi / 2,
+        gt_items.xyz,
+        gt_items.wlh,
+        gt_items.yaw + np.pi / 2,
         threshold_in_overlap_ratio=threshold_in_overlap_ratio,
     )
 
@@ -351,27 +359,27 @@ def detection_iou(
         :param threshold: IOU threshold to consider detection as valid. Must be in [0, 1].
     """
 
-    gt_items = process_dataitem(groundtruth)
-    pred_items = process_dataitem(prediction)
+    gt_items = process_cuboids(groundtruth)
+    pred_items = process_cuboids(prediction)
 
     meter_2d = []
     meter_3d = []
 
-    if gt_items["xyz"].shape[0] == 0 or pred_items["xyz"].shape[0] == 0:
+    if gt_items.xyz.shape[0] == 0 or pred_items.xyz.shape[0] == 0:
         return np.array([0.0]), np.array([0.0])
 
     iou_3d, iou_2d = compute_outer_iou(
-        gt_items["xyz"],
-        gt_items["wlh"],
-        gt_items["yaw"],
-        pred_items["xyz"],
-        pred_items["wlh"],
-        pred_items["yaw"],
+        gt_items.xyz,
+        gt_items.wlh,
+        gt_items.yaw,
+        pred_items.xyz,
+        pred_items.wlh,
+        pred_items.yaw,
     )
 
     for i, m in enumerate(iou_3d.max(axis=1)):
+        j = iou_3d[i].argmax()
         if m >= threshold_in_overlap_ratio:
-            j = iou_3d[i].argmax()
             meter_3d.append(iou_3d[i, j])
             meter_2d.append(iou_2d[i, j])