From ae7543c10ddd511745a445455133ae76f9dca086 Mon Sep 17 00:00:00 2001 From: Sasha Harrison Date: Tue, 26 Apr 2022 13:51:22 -0700 Subject: [PATCH] new GT deletion endpoint --- nucleus/dataset.py | 11 +++++ nucleus/metrics/cuboid_metrics.py | 56 ++++++++++++++++++------ nucleus/metrics/cuboid_utils.py | 72 +++++++++++++++++-------------- 3 files changed, 94 insertions(+), 45 deletions(-) diff --git a/nucleus/dataset.py b/nucleus/dataset.py index a7e7dc9e..5e215a53 100644 --- a/nucleus/dataset.py +++ b/nucleus/dataset.py @@ -1181,6 +1181,17 @@ def delete_annotations( ) return AsyncJob.from_json(response, self._client) + def delete_annotation_refloc( + self, + reference_id: str, + annotation_id: str, + ): + return self._client.make_request( + {}, + f"dataset/{self.id}/annotation/{reference_id}/{annotation_id}", + requests_command=requests.delete, + ) + def get_scene(self, reference_id: str) -> Scene: """Fetches a single scene in the dataset by its reference ID. diff --git a/nucleus/metrics/cuboid_metrics.py b/nucleus/metrics/cuboid_metrics.py index a6a11d03..274566e6 100644 --- a/nucleus/metrics/cuboid_metrics.py +++ b/nucleus/metrics/cuboid_metrics.py @@ -1,4 +1,5 @@ import sys +import warnings from abc import abstractmethod from typing import List, Optional, Union @@ -10,6 +11,9 @@ from .filtering import ListOfAndFilters, ListOfOrAndFilters from .filters import confidence_filter +DEFAULT_IOU_THRESHOLD = 0.1 +DEFAULT_CONFIDENCE_THRESHOLD = 0.0 + class CuboidMetric(Metric): """Abstract class for metrics of cuboids. @@ -27,8 +31,9 @@ class CuboidMetric(Metric): def __init__( self, + iou_threshold: float, enforce_label_match: bool = False, - confidence_threshold: float = 0.0, + confidence_threshold: Optional[float] = None, annotation_filters: Optional[ Union[ListOfOrAndFilters, ListOfAndFilters] ] = None, @@ -54,6 +59,11 @@ def __init__( (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines these filters as a disjunction (OR). """ + if not confidence_threshold: + confidence_threshold = DEFAULT_CONFIDENCE_THRESHOLD + warnings.warn( + f"Got confidence_threshold value of `None`. In this case, we set the confidence_threshold to {confidence_threshold} (include all predictions, regardless of confidence). Consider specifying this value explicitly during metric initialization" + ) self.enforce_label_match = enforce_label_match assert 0 <= confidence_threshold <= 1 self.confidence_threshold = confidence_threshold @@ -99,8 +109,8 @@ class CuboidIOU(CuboidMetric): def __init__( self, enforce_label_match: bool = True, - iou_threshold: float = 0.0, - confidence_threshold: float = 0.0, + iou_threshold: Optional[float] = None, + confidence_threshold: Optional[float] = None, iou_2d: bool = False, annotation_filters: Optional[ Union[ListOfOrAndFilters, ListOfAndFilters] @@ -127,6 +137,11 @@ def __init__( interpreted as a conjunction (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines these filters as a disjunction (OR). """ + if not iou_threshold: + iou_threshold = DEFAULT_IOU_THRESHOLD + warnings.warn( + f"The IoU threshold used for matching was initialized to `None`. In this case, the value of iou_threshold defaults to {iou_threshold}. If this values will produce unexpected behavior, consider specifying the iou_threshold argument during metric initialization" + ) assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." @@ -134,6 +149,7 @@ def __init__( self.iou_2d = iou_2d super().__init__( enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, confidence_threshold=confidence_threshold, annotation_filters=annotation_filters, prediction_filters=prediction_filters, @@ -147,14 +163,16 @@ def eval( iou_3d_metric, iou_2d_metric = detection_iou( predictions, annotations, - threshold_in_overlap_ratio=self.iou_threshold, + self.iou_threshold, ) - weight = max(len(annotations), len(predictions)) + # If there are zero IoU matches, avg_iou defaults to value 0 if self.iou_2d: - avg_iou = iou_2d_metric.sum() / max(weight, sys.float_info.epsilon) + weight = len(iou_2d_metric) + avg_iou = iou_2d_metric.sum() / weight if weight > 0 else 0.0 else: - avg_iou = iou_3d_metric.sum() / max(weight, sys.float_info.epsilon) + weight = len(iou_3d_metric) + avg_iou = iou_3d_metric.sum() / weight if weight > 0 else 0.0 return ScalarResult(avg_iou, weight) @@ -166,8 +184,8 @@ class CuboidPrecision(CuboidMetric): def __init__( self, enforce_label_match: bool = True, - iou_threshold: float = 0.0, - confidence_threshold: float = 0.0, + iou_threshold: Optional[float] = None, + confidence_threshold: Optional[float] = None, annotation_filters: Optional[ Union[ListOfOrAndFilters, ListOfAndFilters] ] = None, @@ -192,12 +210,18 @@ def __init__( interpreted as a conjunction (AND), forming a more selective and multiple column predicate. Finally, the most outer list combines these filters as a disjunction (OR). """ + if not iou_threshold: + iou_threshold = DEFAULT_IOU_THRESHOLD + warnings.warn( + f"The IoU threshold used for matching was initialized to `None`. In this case, the value of iou_threshold defaults to {iou_threshold}. If this values will produce unexpected behavior, consider specifying the iou_threshold argument during metric initialization" + ) assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold super().__init__( enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, confidence_threshold=confidence_threshold, annotation_filters=annotation_filters, prediction_filters=prediction_filters, @@ -211,7 +235,7 @@ def eval( stats = recall_precision( predictions, annotations, - threshold_in_overlap_ratio=self.iou_threshold, + self.iou_threshold, ) weight = stats["tp_sum"] + stats["fp_sum"] precision = stats["tp_sum"] / max(weight, sys.float_info.epsilon) @@ -225,8 +249,8 @@ class CuboidRecall(CuboidMetric): def __init__( self, enforce_label_match: bool = True, - iou_threshold: float = 0.0, - confidence_threshold: float = 0.0, + iou_threshold: Optional[float] = None, + confidence_threshold: Optional[float] = None, annotation_filters: Optional[ Union[ListOfOrAndFilters, ListOfAndFilters] ] = None, @@ -241,12 +265,18 @@ def __init__( iou_threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. Default 0.0 confidence_threshold: minimum confidence threshold for predictions. Must be in [0, 1]. Default 0.0 """ + if not iou_threshold: + iou_threshold = DEFAULT_IOU_THRESHOLD + warnings.warn( + f"The IoU threshold used for matching was initialized to `None`. In this case, the value of iou_threshold defaults to {iou_threshold}. If this values will produce unexpected behavior, consider specifying the iou_threshold argument during metric initialization" + ) assert ( 0 <= iou_threshold <= 1 ), "IoU threshold must be between 0 and 1." self.iou_threshold = iou_threshold super().__init__( enforce_label_match=enforce_label_match, + iou_threshold=iou_threshold, confidence_threshold=confidence_threshold, annotation_filters=annotation_filters, prediction_filters=prediction_filters, @@ -260,7 +290,7 @@ def eval( stats = recall_precision( predictions, annotations, - threshold_in_overlap_ratio=self.iou_threshold, + self.iou_threshold, ) weight = stats["tp_sum"] + stats["fn_sum"] recall = stats["tp_sum"] / max(weight, sys.float_info.epsilon) diff --git a/nucleus/metrics/cuboid_utils.py b/nucleus/metrics/cuboid_utils.py index e23821de..c13575b8 100644 --- a/nucleus/metrics/cuboid_utils.py +++ b/nucleus/metrics/cuboid_utils.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from functools import wraps from typing import Dict, List, Tuple @@ -35,6 +36,14 @@ def __init__(self, *args, **kwargs): from .base import ScalarResult +@dataclass +class ProcessedCuboids: + xyz: np.array + wlh: np.array + yaw: np.array + labels: List[str] + + def group_cuboids_by_label( annotations: List[CuboidAnnotation], predictions: List[CuboidPrediction], @@ -101,19 +110,19 @@ def wrapper( return wrapper -def process_dataitem(dataitem): - processed_item = {} - processed_item["xyz"] = np.array( - [[ann.position.x, ann.position.y, ann.position.z] for ann in dataitem] +def process_cuboids(item_list): + xyz = np.array( + [[ann.position.x, ann.position.y, ann.position.z] for ann in item_list] ) - processed_item["wlh"] = np.array( + wlh = np.array( [ [ann.dimensions.x, ann.dimensions.y, ann.dimensions.z] - for ann in dataitem + for ann in item_list ] ) - processed_item["yaw"] = np.array([ann.yaw for ann in dataitem]) - return processed_item + yaw = np.array([ann.yaw for ann in item_list]) + labels = [ann.label for ann in item_list] + return ProcessedCuboids(xyz, wlh, yaw, labels) def compute_outer_iou( @@ -178,7 +187,6 @@ def compute_outer_iou( .intersection(polygon_1) .area ) - intersection = height_intersection * area_intersection area_0 = wlh_0[:, 0] * wlh_0[:, 1] area_1 = wlh_1[:, 0] * wlh_1[:, 1] @@ -294,23 +302,23 @@ def recall_precision( num_predicted = 0 num_instances = 0 - gt_items = process_dataitem(groundtruth) - pred_items = process_dataitem(prediction) + gt_items = process_cuboids(groundtruth) + pred_items = process_cuboids(prediction) - num_predicted += pred_items["xyz"].shape[0] - num_instances += gt_items["xyz"].shape[0] + num_predicted += pred_items.xyz.shape[0] + num_instances += gt_items.xyz.shape[0] - tp = np.zeros(pred_items["xyz"].shape[0]) - fp = np.ones(pred_items["xyz"].shape[0]) - fn = np.ones(gt_items["xyz"].shape[0]) + tp = np.zeros(pred_items.xyz.shape[0]) + fp = np.ones(pred_items.xyz.shape[0]) + fn = np.ones(gt_items.xyz.shape[0]) mapping = associate_cuboids_on_iou( - pred_items["xyz"], - pred_items["wlh"], - pred_items["yaw"] + np.pi / 2, - gt_items["xyz"], - gt_items["wlh"], - gt_items["yaw"] + np.pi / 2, + pred_items.xyz, + pred_items.wlh, + pred_items.yaw + np.pi / 2, + gt_items.xyz, + gt_items.wlh, + gt_items.yaw + np.pi / 2, threshold_in_overlap_ratio=threshold_in_overlap_ratio, ) @@ -351,27 +359,27 @@ def detection_iou( :param threshold: IOU threshold to consider detection as valid. Must be in [0, 1]. """ - gt_items = process_dataitem(groundtruth) - pred_items = process_dataitem(prediction) + gt_items = process_cuboids(groundtruth) + pred_items = process_cuboids(prediction) meter_2d = [] meter_3d = [] - if gt_items["xyz"].shape[0] == 0 or pred_items["xyz"].shape[0] == 0: + if gt_items.xyz.shape[0] == 0 or pred_items.xyz.shape[0] == 0: return np.array([0.0]), np.array([0.0]) iou_3d, iou_2d = compute_outer_iou( - gt_items["xyz"], - gt_items["wlh"], - gt_items["yaw"], - pred_items["xyz"], - pred_items["wlh"], - pred_items["yaw"], + gt_items.xyz, + gt_items.wlh, + gt_items.yaw, + pred_items.xyz, + pred_items.wlh, + pred_items.yaw, ) for i, m in enumerate(iou_3d.max(axis=1)): + j = iou_3d[i].argmax() if m >= threshold_in_overlap_ratio: - j = iou_3d[i].argmax() meter_3d.append(iou_3d[i, j]) meter_2d.append(iou_2d[i, j])