Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add id fields to Annotations and Predictions for traceability #288

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Add id fields to Annotations and Predictions for traceability #288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
42d6063
Drop invalid polygons for IOU matching
gatli Apr 25, 2022
ff79b7a
Change log for invalid itemsm
gatli Apr 26, 2022
2f3e27d
Add id fields to annotations and predictions when loading from server
gatli Apr 26, 2022
9ad3bd3
Add id fields to docstrings
gatli Apr 26, 2022
5906e6e
Change id to property and init in common Annotation.from_json
gatli Apr 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 20 additions & 47 deletions nucleus/annotation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
EMBEDDING_VECTOR_KEY,
GEOMETRY_KEY,
HEIGHT_KEY,
ID_KEY,
INDEX_KEY,
KEYPOINTS_KEY,
KEYPOINTS_NAMES_KEY,
Expand Down Expand Up @@ -44,12 +45,16 @@

class Annotation:
"""Internal base class, not to be used directly.

.. todo ::
Inherit common constructor parameters from here
"""

reference_id: str
_nucleus_id: Optional[str] = None

@property
def id(self) -> Optional[str]:
return self._nucleus_id

@classmethod
def from_json(cls, payload: dict):
Expand All @@ -65,7 +70,12 @@ def from_json(cls, payload: dict):
}
type_key = payload.get(TYPE_KEY, None)
AnnotationCls = type_key_to_type.get(type_key, SegmentationAnnotation)
return AnnotationCls.from_json(payload)
instance = AnnotationCls.from_json(payload)
# NOTE: Accessing protected var of sub-class looks like the cleanest way for a common classmethod functionality
instance._nucleus_id = payload.get( # pylint: disable=protected-access
ID_KEY, None
)
return instance

def to_payload(self) -> dict:
"""Serializes annotation object to schematized JSON dict."""
Expand All @@ -80,7 +90,6 @@ def to_json(self) -> str:

def has_local_files_to_upload(self) -> bool:
"""Returns True if annotation has local files that need to be uploaded.

Nearly all subclasses have no local files, so we default this to just return
false. If the subclass has local files, it should override this method (but
that is not the only thing required to get local upload of files to work.)
Expand All @@ -91,11 +100,8 @@ def has_local_files_to_upload(self) -> bool:
@dataclass # pylint: disable=R0902
class BoxAnnotation(Annotation): # pylint: disable=R0902
"""A bounding box annotation.

::

from nucleus import BoxAnnotation

box = BoxAnnotation(
label="car",
x=0,
Expand All @@ -107,7 +113,6 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902
metadata={"vehicle_color": "red"},
embedding_vector=[0.1423, 1.432, ...3.829],
)

Parameters:
label (str): The label for this annotation.
x (Union[float, int]): The distance, in pixels, between the left border
Expand All @@ -132,6 +137,7 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902
embedding_vector: Custom embedding vector for this object annotation.
If any custom object embeddings have been uploaded previously to this dataset,
this vector must match the dimensions of the previously ingested vectors.

"""

label: str
Expand Down Expand Up @@ -184,7 +190,6 @@ def to_payload(self) -> dict:
@dataclass
class Point:
"""A point in 2D space.

Parameters:
x (float): The x coordinate of the point.
y (float): The y coordinate of the point.
Expand All @@ -206,19 +211,15 @@ class LineAnnotation(Annotation):
"""A polyline annotation consisting of an ordered list of 2D points.
A LineAnnotation differs from a PolygonAnnotation by not forming a closed
loop, and by having zero area.

::

from nucleus import LineAnnotation

line = LineAnnotation(
label="face",
vertices=[Point(100, 100), Point(200, 300), Point(300, 200)],
reference_id="person_image_1",
annotation_id="person_image_1_line_1",
metadata={"camera_mode": "portrait"},
)

Parameters:
label (str): The label for this annotation.
vertices (List[:class:`Point`]): The list of points making up the line.
Expand All @@ -232,6 +233,7 @@ class LineAnnotation(Annotation):
attach to this annotation. Strings, floats and ints are supported best
by querying and insights features within Nucleus. For more details see
our `metadata guide <https://nucleus.scale.com/docs/upload-metadata>`_.

"""

label: str
Expand Down Expand Up @@ -286,11 +288,8 @@ def to_payload(self) -> dict:
@dataclass
class PolygonAnnotation(Annotation):
"""A polygon annotation consisting of an ordered list of 2D points.

::

from nucleus import PolygonAnnotation

polygon = PolygonAnnotation(
label="bus",
vertices=[Point(100, 100), Point(150, 200), Point(200, 100)],
Expand All @@ -299,7 +298,6 @@ class PolygonAnnotation(Annotation):
metadata={"vehicle_color": "yellow"},
embedding_vector=[0.1423, 1.432, ...3.829],
)

Parameters:
label (str): The label for this annotation.
vertices (List[:class:`Point`]): The list of points making up the polygon.
Expand All @@ -316,6 +314,7 @@ class PolygonAnnotation(Annotation):
embedding_vector: Custom embedding vector for this object annotation.
If any custom object embeddings have been uploaded previously to this dataset,
this vector must match the dimensions of the previously ingested vectors.

"""

label: str
Expand Down Expand Up @@ -373,13 +372,11 @@ def to_payload(self) -> dict:
@dataclass
class Keypoint:
"""A 2D point that has an additional visibility flag.

Keypoints are intended to be part of a larger collection, and connected
via a pre-defined skeleton. A keypoint in this skeleton may be visible
or not-visible, and may be unlabeled and not visible. Because of this,
the x, y coordinates may be optional, assuming that the keypoint is not
visible, and would not be shown as part of the combined label.

Parameters:
x (Optional[float]): The x coordinate of the point.
y (Optional[float]): The y coordinate of the point.
Expand Down Expand Up @@ -417,11 +414,8 @@ class KeypointsAnnotation(Annotation):
"""A keypoints annotation containing a list of keypoints and the structure
of those keypoints: the naming of each point and the skeleton that connects
those keypoints.

::

from nucleus import KeypointsAnnotation

keypoints = KeypointsAnnotation(
label="face",
keypoints=[Keypoint(100, 100), Keypoint(120, 120), Keypoint(visible=False), Keypoint(0, 0)],
Expand All @@ -431,7 +425,6 @@ class KeypointsAnnotation(Annotation):
annotation_id="image_2_face_keypoints_1",
metadata={"face_direction": "forward"},
)

Parameters:
label (str): The label for this annotation.
keypoints (List[:class:`Keypoint`]): The list of keypoints objects.
Expand All @@ -448,6 +441,7 @@ class KeypointsAnnotation(Annotation):
attach to this annotation. Strings, floats and ints are supported best
by querying and insights features within Nucleus. For more details see
our `metadata guide <https://nucleus.scale.com/docs/upload-metadata>`_.

"""

label: str
Expand Down Expand Up @@ -504,7 +498,6 @@ def to_payload(self) -> dict:
@dataclass
class Point3D:
"""A point in 3D space.

Parameters:
x (float): The x coordinate of the point.
y (float): The y coordinate of the point.
Expand All @@ -526,11 +519,8 @@ def to_payload(self) -> dict:
@dataclass # pylint: disable=R0902
class CuboidAnnotation(Annotation): # pylint: disable=R0902
"""A 3D Cuboid annotation.

::

from nucleus import CuboidAnnotation

cuboid = CuboidAnnotation(
label="car",
position=Point3D(100, 100, 10),
Expand All @@ -540,7 +530,6 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902
annotation_id="pointcloud_1_car_cuboid_1",
metadata={"vehicle_color": "green"}
)

Parameters:
label (str): The label for this annotation.
position (:class:`Point3D`): The point at the center of the cuboid
Expand All @@ -555,6 +544,7 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902
annotation. Strings, floats and ints are supported best by querying
and insights features within Nucleus. For more details see our `metadata
guide <https://nucleus.scale.com/docs/upload-metadata>`_.

"""

label: str
Expand Down Expand Up @@ -596,22 +586,17 @@ def to_payload(self) -> dict:
payload[ANNOTATION_ID_KEY] = self.annotation_id
if self.metadata:
payload[METADATA_KEY] = self.metadata

return payload


@dataclass
class Segment:
"""Segment represents either a class or an instance depending on the task type.

For semantic segmentation, this object should store the mapping between a single
class index and the string label.

For instance segmentation, you can use this class to store the label of a single
instance, whose extent in the image is represented by the value of ``index``.

In both cases, additional metadata can be attached to the segment.

Parameters:
label (str): The label name of the class for the class or instance
represented by index in the associated mask.
Expand All @@ -620,6 +605,7 @@ class index and the string label.
Strings, floats and ints are supported best by querying and insights
features within Nucleus. For more details see our `metadata guide
<https://nucleus.scale.com/docs/upload-metadata>`_.

"""

label: str
Expand Down Expand Up @@ -647,21 +633,16 @@ def to_payload(self) -> dict:
@dataclass
class SegmentationAnnotation(Annotation):
"""A segmentation mask on a 2D image.

When uploading a mask annotation, Nucleus expects the mask file to be in
PNG format with each pixel being a 0-255 uint8. Currently, Nucleus only
supports uploading masks from URL.

Nucleus automatically enforces the constraint that each DatasetItem can
have at most one ground truth segmentation mask. As a consequence, if
during upload a duplicate mask is detected for a given image, by default it
will be ignored. You can change this behavior by setting ``update = True``,
which will replace the existing segmentation mask with the new mask.

::

from nucleus import SegmentationAnnotation

segmentation = SegmentationAnnotation(
mask_url="s3://your-bucket-name/segmentation-masks/image_2_mask_id1.png",
annotations=[
Expand All @@ -673,19 +654,16 @@ class SegmentationAnnotation(Annotation):
reference_id="image_2",
annotation_id="image_2_mask_1",
)

Parameters:
mask_url (str): A URL pointing to the segmentation prediction mask which is
accessible to Scale, or a local path. The mask is an HxW int8 array saved in PNG format,
with each pixel value ranging from [0, N), where N is the number of
possible classes (for semantic segmentation) or instances (for instance
segmentation).

The height and width of the mask must be the same as the
original image. One example for semantic segmentation: the mask is 0
for pixels where there is background, 1 where there is a car, and 2
where there is a pedestrian.

Another example for instance segmentation: the mask is 0 for one car,
1 for another car, 2 for a motorcycle and 3 for another motorcycle.
The class name for each value in the mask is stored in the list of
Expand All @@ -703,6 +681,7 @@ class SegmentationAnnotation(Annotation):
is passed to :meth:`Dataset.annotate`, in which case it will be overwritten.
Storing a custom ID here may be useful in order to tie this annotation
to an external database, and its value will be returned for any export.

"""

mask_url: str
Expand Down Expand Up @@ -738,9 +717,7 @@ def to_payload(self) -> dict:
ANNOTATION_ID_KEY: self.annotation_id,
# METADATA_KEY: self.metadata, # TODO(sc: 422637)
}

payload[REFERENCE_ID_KEY] = self.reference_id

return payload

def has_local_files_to_upload(self) -> bool:
Expand Down Expand Up @@ -777,18 +754,14 @@ class AnnotationTypes(Enum):
@dataclass
class CategoryAnnotation(Annotation):
"""A category annotation.

::

from nucleus import CategoryAnnotation

category = CategoryAnnotation(
label="dress",
reference_id="image_1",
taxonomy_name="clothing_type",
metadata={"dress_color": "navy"}
)

Parameters:
label (str): The label for this annotation.
reference_id (str): User-defined ID of the image to which to apply this annotation.
Expand All @@ -798,6 +771,7 @@ class CategoryAnnotation(Annotation):
Strings, floats and ints are supported best by querying and insights
features within Nucleus. For more details see our `metadata guide
<https://nucleus.scale.com/docs/upload-metadata>`_.

"""

label: str
Expand Down Expand Up @@ -890,7 +864,6 @@ def add_annotations(self, annotations: List[Annotation]):
assert isinstance(
annotation, Annotation
), "Expected annotation to be of type 'Annotation"

if isinstance(annotation, BoxAnnotation):
self.box_annotations.append(annotation)
elif isinstance(annotation, LineAnnotation):
Expand Down
35 changes: 35 additions & 0 deletions nucleus/metrics/polygon_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import logging
import sys
from functools import wraps
from typing import Dict, List, Tuple, TypeVar
Expand Down Expand Up @@ -82,6 +83,40 @@ def _iou_assignments_for_same_reference_id(
polygon_annotations = list(map(polygon_annotation_to_shape, annotations))
polygon_predictions = list(map(polygon_annotation_to_shape, predictions))

invalid_anns = [
ann
for ann, poly in zip(annotations, polygon_annotations)
if not poly.is_valid
]
invalid_preds = [
pred
for pred, poly in zip(predictions, polygon_predictions)
if not poly.is_valid
]

if invalid_anns or invalid_preds:
# Filter out invalid polys
polygon_annotations = [
poly
for ann, poly in zip(annotations, polygon_annotations)
if poly.is_valid
]
polygon_predictions = [
poly
for pred, poly in zip(predictions, polygon_predictions)
if poly.is_valid
]
invalid_dataset_ids = set(
ann.reference_id for ann in invalid_anns
).union(set(pred.reference_id for pred in invalid_preds))
# TODO(gunnar): change to .id once field is surfaced)
logging.warning(
"Invalid polygons for dataset items: %s Annotations:%s, predictions: %s",
invalid_dataset_ids,
[a.id for a in invalid_anns],
[p.id for p in invalid_preds],
)

# Compute IoU matrix and set IoU values below the threshold to 0.
iou_matrix = _iou_matrix(polygon_annotations, polygon_predictions)
iou_matrix[iou_matrix < iou_threshold] = 0
Expand Down
Loading