diff --git a/.github/workflows/test-push.yml b/.github/workflows/test-push.yml index 1bf9b2438c..6e28ebe22d 100644 --- a/.github/workflows/test-push.yml +++ b/.github/workflows/test-push.yml @@ -149,3 +149,5 @@ jobs: oauth_client_secret: ${{ secrets.GDRIVE_CLIENT_SECRET }} oauth_refresh_token: ${{ secrets.GDRIVE_REFRESH_TOKEN }} sonar_token: ${{ secrets.SONAR_TOKEN }} + labelbox_token: ${{ secrets.LABELBOX_TOKEN }} + diff --git a/deeplake/integrations/labelbox/__init__.py b/deeplake/integrations/labelbox/__init__.py new file mode 100644 index 0000000000..26c8c46656 --- /dev/null +++ b/deeplake/integrations/labelbox/__init__.py @@ -0,0 +1,9 @@ +from deeplake.integrations.labelbox.labelbox_ import ( + create_labelbox_annotation_project, + create_dataset_from_video_annotation_project, + create_dataset_from_video_annotation_project_with_custom_data_filler, + converter_for_video_project_with_id, +) +from deeplake.integrations.labelbox.labelbox_azure_utils import ( + load_blob_file_paths_from_azure, +) diff --git a/deeplake/integrations/labelbox/labelbox_.py b/deeplake/integrations/labelbox/labelbox_.py new file mode 100644 index 0000000000..65a0a61c85 --- /dev/null +++ b/deeplake/integrations/labelbox/labelbox_.py @@ -0,0 +1,399 @@ +import deeplake +import os +import labelbox as lb # type: ignore +import uuid + +from deeplake.integrations.labelbox.labelbox_utils import * +from deeplake.integrations.labelbox.labelbox_converter import labelbox_video_converter +from deeplake.integrations.labelbox.v3_converters import * +from deeplake.integrations.labelbox.labelbox_metadata_utils import * + + +def converter_for_video_project_with_id( + project_id, + client, + deeplake_ds_loader, + lb_api_key, + group_mapping=None, + fail_on_error=False, + fail_on_labelbox_project_export_error=False, + generate_metadata=True, + metadata_prefix="metadata", +) -> Optional[labelbox_video_converter]: + """ + Creates a converter for Labelbox video project to a Deeplake dataset format based on annotation types. + + Args: + project_id (str): The unique identifier for the Labelbox project to convert. + client (LabelboxClient): An authenticated Labelbox client instance for API access. + deeplake_ds_loader (callable): A function that creates/loads a Deeplake dataset given a name. + lb_api_key (str): Labelbox API key for authentication. + group_mapping (dict, optional): A dictionary mapping annotation kinds (labelbox_kind) to the desired tensor group name (tensor_name). This mapping determines whether annotations of the same kind should be grouped into the same tensor or kept separate. + fail_on_error (bool, optional): Whether to raise an exception if data validation fails. Defaults to False. + fail_on_labelbox_project_export_error (bool, optional): Whether to raise an exception if Labelbox project export fails. Defaults to False. + generate_metadata (bool, optional): Whether to generate metadata tensors. Defaults to True. + metadata_prefix (str, optional): Prefix for metadata tensors. Defaults to "metadata". Will be ignored if generate_metadata is False. + + + Returns: + Optional[labelbox_video_converter]: Returns a labelbox_type_converter if successful, None if no data is found. + The returned converter can be used to apply Labelbox annotations to a Deeplake dataset. + + Raises: + Exception: If project data validation fails. + + Example: + >>> client = LabelboxClient(api_key='your_api_key') + >>> converter = converter_for_video_project_with_id( + ... '', + ... client, + ... lambda name: deeplake.load(name), + ... 'your_api_key', + ... group_mapping={"raster-segmentation": "mask"} + ... ) + >>> if converter: + ... # Use converter to apply annotations + ... ds = converter.dataset_with_applied_annotations() + + Notes: + - Supports Video ontology from labelbox. + - The function first validates the project data before setting up converters. + """ + project_json = labelbox_get_project_json_with_id_( + client, project_id, fail_on_labelbox_project_export_error + ) + + if len(project_json) == 0: + print("no data") + return None + + ds_name = project_json[0]["projects"][project_id]["name"] + deeplake_dataset = deeplake_ds_loader(ds_name) + + if not validate_project_data_(project_json, deeplake_dataset, project_id, "video"): + if fail_on_error: + raise Exception("Data validation failed") + + ontology_id = project_json[0]["projects"][project_id]["project_details"][ + "ontology_id" + ] + ontology = client.get_ontology(ontology_id) + + converters = { + "rectangle": bbox_converter_, + "radio": radio_converter_, + "checklist": checkbox_converter_, + "point": point_converter_, + "line": line_converter_, + "raster-segmentation": raster_segmentation_converter_, + "text": text_converter_, + } + + if generate_metadata: + tensor_name_generator = lambda name: ( + f"{metadata_prefix}/{name}" if metadata_prefix else name + ) + + metadata_generators = { + tensor_name_generator("name"): { + "generator": get_video_name_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("data_row_id"): { + "generator": get_data_row_id_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("label_creator"): { + "generator": get_label_creator_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("frame_rate"): { + "generator": get_frame_rate_from_video_project_, + "create_tensor_kwargs": {"htype": "generic", "dtype": "int32"}, + }, + tensor_name_generator("frame_count"): { + "generator": get_frame_count_from_video_project_, + "create_tensor_kwargs": {"htype": "generic", "dtype": "int32"}, + }, + tensor_name_generator("width"): { + "generator": get_width_from_video_project_, + "create_tensor_kwargs": {"htype": "generic", "dtype": "int32"}, + }, + tensor_name_generator("height"): { + "generator": get_height_from_video_project_, + "create_tensor_kwargs": {"htype": "generic", "dtype": "int32"}, + }, + tensor_name_generator("ontology_id"): { + "generator": get_ontology_id_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("project_name"): { + "generator": get_project_name_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("dataset_name"): { + "generator": get_dataset_name_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("dataset_id"): { + "generator": get_dataset_id_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("global_key"): { + "generator": get_global_key_from_video_project_, + "create_tensor_kwargs": {"htype": "text"}, + }, + tensor_name_generator("frame_number"): { + "generator": lambda project, ctx: ctx["frame_idx"], + "create_tensor_kwargs": {"htype": "generic", "dtype": "int32"}, + }, + tensor_name_generator("current_frame_name"): { + "generator": lambda project, ctx: f"{get_video_name_from_video_project_(project, ctx)}_{ctx['frame_idx']:06d}", + "create_tensor_kwargs": {"htype": "text"}, + }, + } + else: + metadata_generators = None + + return labelbox_video_converter( + ontology, + converters, + project_json, + project_id, + deeplake_dataset, + {"ds": deeplake_dataset, "lb_api_key": lb_api_key}, + metadata_generators=metadata_generators, + group_mapping=group_mapping, + ) + + +def create_labelbox_annotation_project( + video_paths, + lb_dataset_name, + lb_project_name, + lb_client, + lb_ontology=None, + lb_batch_priority=5, + data_upload_strategy="fail", + lb_batches_name=None, +): + """ + Creates labelbox dataset for video annotation and sets up corresponding Labelbox project. + + Args: + video_paths (List[str]): List of paths to video files to be processed can be either all local or all pre-signed remote. + lb_dataset_name (str): Name for Labelbox dataset. + lb_project_name (str): Name for Labelbox project. + lb_client (LabelboxClient): Authenticated Labelbox client instance + lb_ontology (Ontology, optional): Labelbox ontology to connect to project. Defaults to None + lb_batch_priority (int, optional): Priority for Labelbox batches. Defaults to 5 + data_upload_strategy (str, optional): Strategy for uploading data to Labelbox. Can be 'fail', 'skip', or 'all'. Defaults to 'fail' + lb_batches_name (str, optional): Name for Labelbox batches. Defaults to None. If None, will use lb_dataset_name + '_batch-' + """ + + video_paths = filter_video_paths_(video_paths, data_upload_strategy) + + assets = video_paths + + # validate paths + all_local = [os.path.exists(p) for p in video_paths] + if any(all_local) and not all(all_local): + raise Exception(f"video paths must be all local or all remote: {video_paths}") + + if len(all_local): + if not all_local[0]: + assets = [ + { + "row_data": p, + "global_key": str(uuid.uuid4()), + "media_type": "VIDEO", + "metadata_fields": [], + "attachments": [], + } + for p in video_paths + ] + + print("uploading videos to labelbox") + lb_ds = lb_client.create_dataset(name=lb_dataset_name) + task = lb_ds.create_data_rows(assets) + task.wait_till_done() + + if task.errors: + raise Exception(f"failed to upload videos to labelbox: {task.errors}") + + if len(all_local): + if all_local[0]: + print("assigning global keys to data rows") + rows = [ + { + "data_row_id": lb_ds.data_row_for_external_id(p).uid, + "global_key": str(uuid.uuid4()), + } + for p in video_paths + ] + res = lb_client.assign_global_keys_to_data_rows(rows) + if res["status"] != "SUCCESS": + raise Exception(f"failed to assign global keys to data rows: {res}") + + print("successfuly uploaded videos to labelbox") + + # Create a new project + project = lb_client.create_project( + name=lb_project_name, media_type=lb.MediaType.Video + ) + + if lb_batches_name is None: + lb_batches_name = lb_dataset_name + "_batch-" + + task = project.create_batches_from_dataset( + name_prefix=lb_batches_name, dataset_id=lb_ds.uid, priority=lb_batch_priority + ) + + if task.errors(): + raise Exception(f"Error creating batches: {task.errors()}") + + if lb_ontology: + project.connect_ontology(lb_ontology) + + +def create_dataset_from_video_annotation_project_with_custom_data_filler( + deeplake_ds_path, + project_id, + lb_client, + lb_api_key, + data_filler, + deeplake_creds=None, + deeplake_org_id=None, + deeplake_token=None, + overwrite=False, + fail_on_error=False, + url_presigner=None, + video_generator_batch_size=100, + fail_on_labelbox_project_export_error=False, +) -> deeplake.Dataset: + """ + Creates a Deeplake dataset from an existing Labelbox video annotation project using custom data processing. + Downloads video frames from Labelbox and processes them using provided data filler functions. + + Args: + deeplake_ds_path (str): Path where the Deeplake dataset will be created/stored. + Can be local path or cloud path (e.g. 'hub://org/dataset') + project_id (str): Labelbox project ID to import data from + lb_client (LabelboxClient): Authenticated Labelbox client instance + lb_api_key (str): Labelbox API key for accessing video frames + data_filler (dict): Dictionary containing two functions: + - 'create_tensors': callable(ds) -> None + Creates necessary tensors in the dataset + - 'fill_data': callable(ds, group_ids, indexes, frames) -> None + Fills dataset with processed frame batches + deeplake_creds (dict): Dictionary containing credentials for deeplake + deeplake_org_id (str, optional): Organization ID for Deeplake cloud storage. + deeplake_token (str, optional): Authentication token for Deeplake cloud storage. + Required if using hub:// path. Defaults to None + overwrite (bool, optional): Whether to overwrite existing dataset. Defaults to False + fail_on_error (bool, optional): Whether to raise an exception if data validation fails. Defaults to False + url_presigner (callable, optional): Function that takes a URL and returns a pre-signed URL and headers (str, dict). Default will use labelbox access token to access the data. Is useful when used cloud storage integrations. + video_generator_batch_size (int, optional): Number of frames to process in each batch. Defaults to 100 + fail_on_labelbox_project_export_error (bool, optional): Whether to raise an exception if Labelbox project export fails. Defaults to False + + Returns: + Dataset: Created Deeplake dataset containing processed video frames and Labelbox metadata. + Returns empty dataset if no data found in project. + + Notes: + - The function does not fetch the annotations from Labelbox, only the video frames. After creating the dataset, use the converter to apply annotations. + """ + ds = deeplake.empty( + deeplake_ds_path, + overwrite=overwrite, + creds=deeplake_creds, + org_id=deeplake_org_id, + token=deeplake_token, + ) + data_filler["create_tensors"](ds) + + proj = labelbox_get_project_json_with_id_( + lb_client, project_id, fail_on_labelbox_project_export_error + ) + if len(proj) == 0: + print("no data") + return ds + + if not validate_project_creation_data_(proj, project_id, "video"): + if fail_on_error: + raise Exception("Data validation failed") + + video_files = [] + + if url_presigner is None: + + def default_presigner(url): + if lb_api_key is None: + return url, {} + return url, {"headers": {"Authorization": f"Bearer {lb_api_key}"}} + + url_presigner = default_presigner + + for idx, p in enumerate(proj): + video_url = p["data_row"]["row_data"] + header = None + if not os.path.exists(video_url): + if not is_remote_resource_public_(video_url): + video_url, header = url_presigner(video_url) + for frame_indexes, frames in frames_batch_generator_( + video_url, header=header, batch_size=video_generator_batch_size + ): + data_filler["fill_data"](ds, [idx] * len(frames), frame_indexes, frames) + video_files.append(external_url_from_video_project_(p)) + + ds.info["labelbox_meta"] = { + "project_id": project_id, + "type": "video", + "sources": video_files, + "project_name": proj[0]["projects"][project_id]["name"], + } + + ds.commit() + + return ds + + +def create_dataset_from_video_annotation_project( + deeplake_ds_path, + project_id, + lb_client, + lb_api_key, + deeplake_creds=None, + deeplake_org_id=None, + deeplake_token=None, + overwrite=False, + fail_on_error=False, + url_presigner=None, + video_generator_batch_size=100, + fail_on_labelbox_project_export_error=False, +) -> deeplake.Dataset: + """ + See create_dataset_from_video_annotation_project_with_custom_data_filler for complete documentation. + + The only difference is this function uses default tensor creation and data filling functions: + - create_tensors_default_: Creates default tensor structure + - fill_data_default_: Fills tensors with default processing + """ + return create_dataset_from_video_annotation_project_with_custom_data_filler( + deeplake_ds_path, + project_id, + lb_client, + lb_api_key, + data_filler={ + "create_tensors": create_tensors_default_, + "fill_data": fill_data_default_, + }, + deeplake_creds=deeplake_creds, + deeplake_org_id=deeplake_org_id, + deeplake_token=deeplake_token, + overwrite=overwrite, + fail_on_error=fail_on_error, + url_presigner=url_presigner, + video_generator_batch_size=video_generator_batch_size, + fail_on_labelbox_project_export_error=fail_on_labelbox_project_export_error, + ) diff --git a/deeplake/integrations/labelbox/labelbox_azure_utils.py b/deeplake/integrations/labelbox/labelbox_azure_utils.py new file mode 100644 index 0000000000..1524505366 --- /dev/null +++ b/deeplake/integrations/labelbox/labelbox_azure_utils.py @@ -0,0 +1,26 @@ +from azure.storage.blob import BlobServiceClient + + +def load_blob_file_paths_from_azure( + storage_account_name, + container_name, + parent_path, + sas_token, + predicate=lambda x: True, +): + # Construct the account URL with the SAS token + account_url = f"https://{storage_account_name}.blob.core.windows.net" + # Service client to connect to Azure Blob Storage using SAS token + blob_service_client = BlobServiceClient( + account_url=account_url, credential=sas_token + ) + # Get a reference to the container + container_client = blob_service_client.get_container_client(container_name) + # List blobs in the directory + blob_list = container_client.list_blobs(name_starts_with=parent_path) + file_url_list = [ + f"{account_url}/{container_name}/{blob.name}?{sas_token}" + for blob in blob_list + if predicate(blob.name) + ] + return file_url_list diff --git a/deeplake/integrations/labelbox/labelbox_converter.py b/deeplake/integrations/labelbox/labelbox_converter.py new file mode 100644 index 0000000000..dc7cfd1a63 --- /dev/null +++ b/deeplake/integrations/labelbox/labelbox_converter.py @@ -0,0 +1,379 @@ +from deeplake.integrations.labelbox.labelbox_utils import * +import tqdm +from collections import defaultdict + + +class labelbox_type_converter: + def __init__( + self, + ontology, + converters, + project, + project_id, + dataset, + context, + metadata_generators=None, + group_mapping=None, + ): + self.labelbox_feature_id_to_type_mapping = dict() + self.regsistered_actions = dict() + self.label_mappings = dict() + self.values_cache = dict() + self.registered_interpolators = dict() + + self.metadata_generators_ = metadata_generators + + self.project = project + self.project_id = project_id + self.dataset = dataset + + self.group_mapping = group_mapping if group_mapping is not None else dict() + self.groupped_tensor_overrides = dict() + + self.labelbox_type_converters_ = converters + + self.register_ontology_(ontology, context) + + def register_feature_id_for_kind(self, kind, key, obj, tensor_name): + self.labelbox_feature_id_to_type_mapping[obj.feature_schema_id] = { + "kind": kind, + "key": key, + "name": obj.name, + "tensor_name": tensor_name, + } + + def dataset_with_applied_annotations(self): + idx_offset = 0 + print("total annotations projects count: ", len(self.project)) + + if self.metadata_generators_: + self.generate_metadata_tensors_(self.metadata_generators_, self.dataset) + + for p_idx, p in enumerate(self.yield_projects_(self.project, self.dataset)): + if "labels" not in p["projects"][self.project_id]: + print("no labels for project with index: ", p_idx) + continue + print("parsing annotations for project with index: ", p_idx) + for lbl_idx, labels in enumerate(p["projects"][self.project_id]["labels"]): + self.values_cache = dict() + if "frames" not in labels["annotations"]: + continue + frames = labels["annotations"]["frames"] + if not len(frames): + print( + "skip", + external_url_from_video_project_(p), + "with label idx", + lbl_idx, + "as it has no frames", + ) + continue + + assert len(frames) <= p["media_attributes"]["frame_count"] + + print("parsing frames for label index: ", lbl_idx) + for i in tqdm.tqdm(range(p["media_attributes"]["frame_count"])): + if str(i + 1) not in frames: + continue + self.parse_frame_(frames[str(i + 1)], idx_offset + i) + + if "segments" not in labels["annotations"]: + continue + segments = labels["annotations"]["segments"] + # the frames contain only the interpolated values + # iterate over segments and assign same value to all frames in the segment + self.parse_segments_(segments, frames, idx_offset) + + self.apply_cached_values_(self.values_cache, idx_offset) + if self.metadata_generators_: + print("filling metadata for project with index: ", p_idx) + self.fill_metadata_( + self.metadata_generators_, + self.dataset, + p, + self.project_id, + p["media_attributes"]["frame_count"], + ) + + idx_offset += p["media_attributes"]["frame_count"] + + return self.dataset + + def register_tool_(self, tool, context, fix_grouping_only): + if tool.tool.value not in self.labelbox_type_converters_: + print("skip tool:", tool.tool.value) + return + + prefered_name = tool.name + + if tool.tool.value in self.group_mapping: + prefered_name = self.group_mapping[tool.tool.value] + else: + prefered_name = tool.name + + should_group_with_classifications = len(tool.classifications) > 0 + if should_group_with_classifications: + tool_name = prefered_name + "/" + prefered_name + if fix_grouping_only: + if tool.tool.value in self.group_mapping: + self.groupped_tensor_overrides[tool.tool.value] = tool_name + else: + tool_name = prefered_name + + for classification in tool.classifications: + self.register_classification_( + classification, + context, + fix_grouping_only=fix_grouping_only, + parent=prefered_name, + ) + + if fix_grouping_only: + return + + if tool.tool.value in self.groupped_tensor_overrides: + tool_name = self.groupped_tensor_overrides[tool.tool.value] + + self.labelbox_type_converters_[tool.tool.value]( + tool, self, tool_name, context, tool.tool.value in self.group_mapping + ) + + def register_classification_(self, tool, context, fix_grouping_only, parent=""): + if tool.class_type.value not in self.labelbox_type_converters_: + return + + if tool.class_type.value in self.group_mapping: + prefered_name = (parent + "/" if parent else "") + self.group_mapping[ + tool.class_type.value + ] + else: + prefered_name = (parent + "/" if parent else "") + tool.name + + if fix_grouping_only: + return + + self.labelbox_type_converters_[tool.class_type.value]( + tool, + self, + prefered_name, + context, + tool.class_type.value in self.group_mapping, + ) + + def register_ontology_(self, ontology, context, fix_grouping_only=True): + for tool in ontology.tools(): + self.register_tool_(tool, context, fix_grouping_only=fix_grouping_only) + + for classification in ontology.classifications(): + if classification.scope.value != "index": + print("skip global classification:", classification.name) + continue + self.register_classification_( + classification, context, fix_grouping_only=fix_grouping_only + ) + + if fix_grouping_only: + self.register_ontology_(ontology, context, fix_grouping_only=False) + + def parse_frame_(self, frame, idx): + if "objects" in frame: + for _, obj in frame["objects"].items(): + self.parse_object_(obj, idx) + + for obj in frame.get("classifications", []): + self.parse_classification_(obj, idx) + + def parse_object_(self, obj, idx): + if obj["feature_schema_id"] not in self.regsistered_actions: + print("skip object:", obj["feature_schema_id"]) + return + + self.regsistered_actions[obj["feature_schema_id"]](idx, obj) + + for obj in obj.get("classifications", []): + self.parse_classification_(obj, idx) + + def parse_classification_(self, obj, idx): + if obj["feature_schema_id"] not in self.regsistered_actions: + print("skip classification:", obj["feature_schema_id"]) + return + + self.regsistered_actions[obj["feature_schema_id"]](idx, obj) + + for obj in obj.get("classifications", []): + self.parse_classification_(obj, idx) + + def find_object_with_feature_id_(self, frame, feature_id): + if isinstance(frame, list): + for f in frame: + if ret := self.find_object_with_feature_id_(f, feature_id): + return ret + + if "objects" in frame: + if feature_id in frame["objects"]: + return frame["objects"][feature_id] + for _, obj in frame["objects"].items(): + if ret := self.find_object_with_feature_id_(obj, feature_id): + return ret + + if "classifications" in frame: + for obj in frame["classifications"]: + if ret := self.find_object_with_feature_id_(obj, feature_id): + return ret + k = self.labelbox_feature_id_to_type_mapping[obj["feature_schema_id"]][ + "key" + ] + if k in obj: + if ret := self.find_object_with_feature_id_(obj[k], feature_id): + return ret + + if "feature_id" in frame and frame["feature_id"] == feature_id: + return frame + + return None + + def existing_sub_ranges_(self, frames, r, feature_id): + end = r[1] + sub_ranges = [(r[0], end)] + for i in range(r[0] + 1, end): + if str(i) not in frames: + continue + if self.find_object_with_feature_id_(frames[str(i)], feature_id) is None: + continue + sub_ranges[-1] = (sub_ranges[-1][0], i) + sub_ranges.append((i, end)) + return sub_ranges + + def parse_segments_(self, segments, frames, offset): + print("total segments count to parse:", len(segments)) + for feature_id, ranges in segments.items(): + print("parsing segments with feature id: ", feature_id) + for r in tqdm.tqdm(ranges): + sub_ranges = self.existing_sub_ranges_(frames, r, feature_id) + for st, en in sub_ranges: + assert str(st) in frames + + start = self.find_object_with_feature_id_( + frames[str(st)], feature_id + ) + if str(en) in frames: + end = self.find_object_with_feature_id_( + frames[str(en)], feature_id + ) + else: + end = start + + assert start + assert end + assert start["feature_schema_id"] == end["feature_schema_id"] + + for i in range(st + 1, en + 1): + # skip if the frame already has the object + if ( + str(i) in frames + and self.find_object_with_feature_id_( + frames[str(i)], feature_id + ) + is not None + ): + continue + + if start["feature_schema_id"] in self.registered_interpolators: + obj = self.registered_interpolators[ + start["feature_schema_id"] + ](start, end, (i - st) / (en - st)) + else: + obj = end + + self.regsistered_actions[obj["feature_schema_id"]]( + offset + i - 1, obj + ) + # nested classifications are not in the segments + for o in obj.get("classifications", []): + self.regsistered_actions[o["feature_schema_id"]]( + offset + i - 1, o + ) + + def apply_cached_values_(self, cache, offset): + print("applying cached values") + for tensor_name, row_map in cache.items(): + print("applying cached values for tensor: ", tensor_name) + if len(self.dataset[tensor_name]) < offset: + print( + "extending dataset for tensor: ", + tensor_name, + "size: ", + offset - len(self.dataset[tensor_name]), + ) + self.dataset[tensor_name].extend( + [None] * (offset - len(self.dataset[tensor_name])) + ) + max_val = max(row_map.keys()) - offset + values = [] + for i in tqdm.tqdm(range(max_val + 1)): + key = i + offset + if key in row_map: + values.append(row_map[key]) + else: + values.append(None) + + self.dataset[tensor_name].extend(values) + + def yield_projects_(self, project_j, ds): + raise NotImplementedError("fixed_project_order_ is not implemented") + + def generate_metadata_tensors_(self, generators, ds): + for tensor_name, v in generators.items(): + try: + ds.create_tensor(tensor_name, **v["create_tensor_kwargs"]) + except: + pass + + def fill_metadata_(self, generators, dataset, project, project_id, frames_count): + metadata_dict = defaultdict(list) + context = {"project_id": project_id} + for tensor_name, v in generators.items(): + for i in range(frames_count): + context["frame_idx"] = i + metadata_dict[tensor_name].append(v["generator"](project, context)) + + for tensor_name, values in metadata_dict.items(): + dataset[tensor_name].extend(values) + + +# if changes are made to the labelbox_video_converter class, check if ontology_for_debug works correctly +class labelbox_video_converter(labelbox_type_converter): + def __init__( + self, + ontology, + converters, + project, + project_id, + dataset, + context, + metadata_generators=None, + group_mapping=None, + ): + super().__init__( + ontology, + converters, + project, + project_id, + dataset, + context, + metadata_generators, + group_mapping, + ) + + def yield_projects_(self, project_j, ds): + if "labelbox_meta" not in ds.info: + raise ValueError("No labelbox meta data in dataset") + info = ds.info["labelbox_meta"] + + def sorter(p): + url = external_url_from_video_project_(p) + return info["sources"].index(url) + + ordered_values = sorted(project_j, key=sorter) + for p in ordered_values: + yield p diff --git a/deeplake/integrations/labelbox/labelbox_debug.py b/deeplake/integrations/labelbox/labelbox_debug.py new file mode 100644 index 0000000000..523751c5cd --- /dev/null +++ b/deeplake/integrations/labelbox/labelbox_debug.py @@ -0,0 +1,127 @@ +# classes in this file are needed to debug the ontology and labelbox projects when there's no access to the labelbox workspace + + +# helper classes to support same accessors as labelbox instances for accessing the ontology +class ontology_list_for_debug(list): + def __call__(self): + return self + + +class ontology_for_debug: + def __init__(self, data): + for key, value in data.items(): + if isinstance(value, dict): + setattr(self, key, ontology_for_debug(value)) + elif isinstance(value, list): + setattr( + self, + key, + ontology_list_for_debug( + [ + ontology_for_debug(item) if isinstance(item, dict) else item + for item in value + ] + ), + ) + else: + setattr(self, key, value) + + def __call__(self): + return self + + +# Creates ontology object from the final exported labelbox project. +# This function shall replace `client.get_ontology(ontology_id)` in the converter script. +def ontology_for_debug_from_json(projects, project_id): + + global_objects = {} + + classifications = set() + tools = {} + + # handle the rest of the tools if needed + annotation_kind_map = { + "VideoBoundingBox": "rectangle", + } + + def parse_classification_(classification): + d = { + "feature_schema_id": classification["feature_schema_id"], + "name": classification["name"], + "scope": {"value": "index"}, + "options": [], + } + + option = None + + # handle the rest of the tools if needed + if "radio_answer" in classification: + d["class_type"] = {"value": "radio"} + option = { + "name": classification["radio_answer"]["name"], + "value": classification["radio_answer"]["value"], + "feature_schema_id": classification["radio_answer"][ + "feature_schema_id" + ], + } + + if "checkbox_answers" in classification: + d["class_type"] = {"value": "checkbox"} + option = { + "name": classification["checkbox_answers"]["name"], + "value": classification["checkbox_answers"]["value"], + "feature_schema_id": classification["checkbox_answers"][ + "feature_schema_id" + ], + } + + assert option is not None + + if classification["feature_schema_id"] not in global_objects: + global_objects[classification["feature_schema_id"]] = d + + d = global_objects[classification["feature_schema_id"]] + + if option not in d["options"]: + d["options"].append(option) + + return d + + def parse_tool(tool): + tools[tool["feature_schema_id"]] = { + "feature_schema_id": tool["feature_schema_id"], + "name": tool["name"], + "tool": {"value": annotation_kind_map[tool["annotation_kind"]]}, + } + + classifications = [] + for c in tool.get("classifications", []): + parse_classification_(c) + classifications.append(c["feature_schema_id"]) + + tools[tool["feature_schema_id"]]["classifications"] = classifications + + for p in projects: + for label in p["projects"][project_id]["labels"]: + for _, frame in label["annotations"]["frames"].items(): + for f_id, tool in frame["objects"].items(): + parse_tool(tool) + + for classification in frame["classifications"]: + d = parse_classification_(classification) + classifications.add(d["feature_schema_id"]) + + final_tools = list(tools.values()) + + for tool in final_tools: + for idx in range(len(tool["classifications"])): + tool["classifications"][idx] = global_objects[tool["classifications"][idx]] + + final_classifications = [] + + for classification in classifications: + final_classifications.append(global_objects[classification]) + + return ontology_for_debug( + {"classifications": final_classifications, "tools": final_tools} + ) diff --git a/deeplake/integrations/labelbox/labelbox_metadata_utils.py b/deeplake/integrations/labelbox/labelbox_metadata_utils.py new file mode 100644 index 0000000000..51992e6f94 --- /dev/null +++ b/deeplake/integrations/labelbox/labelbox_metadata_utils.py @@ -0,0 +1,90 @@ +import os + + +def get_video_name_from_video_project_(project, ctx): + if "data_row" not in project: + return None + if "external_id" in project["data_row"]: + return os.path.splitext(os.path.basename(project["data_row"]["external_id"]))[0] + if "row_data" in project["data_row"]: + return os.path.splitext(os.path.basename(project["data_row"]["row_data"]))[0] + return None + + +def get_data_row_id_from_video_project_(project, ctx): + try: + return project["data_row"]["id"] + except: + return None + + +def get_label_creator_from_video_project_(project, ctx): + try: + return project["projects"][ctx["project_id"]]["labels"][0]["label_details"][ + "created_by" + ] + except: + return None + + +def get_frame_rate_from_video_project_(project, ctx): + try: + return project["media_attributes"]["frame_rate"] + except: + return None + + +def get_frame_count_from_video_project_(project, ctx): + try: + return project["media_attributes"]["frame_count"] + except: + return None + + +def get_width_from_video_project_(project, ctx): + try: + return project["media_attributes"]["width"] + except: + return None + + +def get_height_from_video_project_(project, ctx): + try: + return project["media_attributes"]["height"] + except: + return None + + +def get_ontology_id_from_video_project_(project, ctx): + try: + return project["projects"][ctx["project_id"]]["project_details"]["ontology_id"] + except: + return None + + +def get_project_name_from_video_project_(project, ctx): + try: + return project["projects"][ctx["project_id"]]["name"] + except: + return None + + +def get_dataset_name_from_video_project_(project, ctx): + try: + return project["data_row"]["details"]["dataset_name"] + except: + return None + + +def get_dataset_id_from_video_project_(project, ctx): + try: + return project["data_row"]["details"]["dataset_id"] + except: + return None + + +def get_global_key_from_video_project_(project, ctx): + try: + return project["data_row"]["details"]["global_key"] + except: + return None diff --git a/deeplake/integrations/labelbox/labelbox_utils.py b/deeplake/integrations/labelbox/labelbox_utils.py new file mode 100644 index 0000000000..193ec46598 --- /dev/null +++ b/deeplake/integrations/labelbox/labelbox_utils.py @@ -0,0 +1,214 @@ +import numpy as np +from typing import Generator, Tuple, Optional, Any +import labelbox as lb # type: ignore +import av +import requests +from collections import Counter + + +def is_remote_resource_public_(url): + try: + response = requests.head(url, allow_redirects=True) + return response.status_code == 200 + except requests.exceptions.RequestException as e: + return False + + +def filter_video_paths_(video_paths, strategy): + if strategy == "all": + return video_paths + unique_paths = set(video_paths) + if strategy == "fail": + if len(unique_paths) != len(video_paths): + counter = Counter(video_paths) + duplicates = [k for k, v in counter.items() if v > 1] + raise ValueError("Duplicate video paths detected: " + ", ".join(duplicates)) + return video_paths + + if strategy == "skip": + if len(unique_paths) != len(video_paths): + counter = Counter(video_paths) + duplicates = [k for k, v in counter.items() if v > 1] + print( + "Duplicate video paths detected, filtering out duplicates: ", duplicates + ) + return list(unique_paths) + + raise ValueError(f"Invalid data upload strategy: {strategy}") + + +def frame_generator_( + video_path: str, header: Optional[dict[str, Any]] = None, retries: int = 5 +) -> Generator[Tuple[int, np.ndarray], None, None]: + """ + Generate frames from a video file. + + Parameters: + video_path (str): Path to the video file + header (dict, optional): Optional request header for authorization + + Yields: + tuple: (frame_number, frame_data) + - frame_number (int): The sequential number of the frame + - frame_data (numpy.ndarray): The frame image data + """ + + def get_video_container(current_retries): + try: + return av.open(video_path, options=header) + except Exception as e: + if current_retries > 0: + print(f"Failed opening video: {e}. Retrying...") + return get_video_container(current_retries - 1) + else: + raise e + + try: + container = get_video_container(retries) + print(f"Start generating frames from {video_path}") + frame_num = 0 + for frame in container.decode(video=0): + yield frame_num, frame.to_ndarray(format="rgb24") + frame_num += 1 + except Exception as e: + print(f"Failed generating frames: {e}") + + +def frames_batch_generator_( + video_path: str, + header: Optional[dict[str, Any]] = None, + batch_size=100, + retries: int = 5, +): + frames, indexes = [], [] + for frame_num, frame in frame_generator_(video_path, header, retries): + frames.append(frame) + indexes.append(frame_num) + if len(frames) < batch_size: + continue + yield indexes, frames + frames, indexes = [], [] + + if len(frames): + yield indexes, frames + + +def external_url_from_video_project_(p): + if "external_id" in p["data_row"]: + return p["data_row"]["external_id"] + return p["data_row"]["row_data"] + + +def validate_video_project_data_impl_(project_j, deeplake_dataset, project_id): + if "labelbox_meta" not in deeplake_dataset.info: + return False + info = deeplake_dataset.info["labelbox_meta"] + + if info["type"] != "video": + return False + + if project_id != info["project_id"]: + return False + + if len(project_j) != len(info["sources"]): + return False + + if len(project_j) == 0: + return True + + ontology_ids = set() + + for p in project_j: + url = external_url_from_video_project_(p) + if url not in info["sources"]: + return False + + ontology_ids.add(p["projects"][project_id]["project_details"]["ontology_id"]) + + if len(ontology_ids) != 1: + return False + + return True + + +PROJECT_DATA_VALIDATION_MAP_ = {"video": validate_video_project_data_impl_} + + +def validate_project_data_(proj, ds, project_id, type): + if type not in PROJECT_DATA_VALIDATION_MAP_: + raise ValueError(f"Invalid project data type: {type}") + return PROJECT_DATA_VALIDATION_MAP_[type](proj, ds, project_id) + + +def validate_video_project_creation_data_impl_(project_j, project_id): + if len(project_j) == 0: + return True + + for p in project_j: + for l in p["projects"][project_id]["labels"]: + if l["label_kind"] != "Video": + return False + + if p["media_attributes"]["asset_type"] != "video": + return False + + return True + + +PROJECT_DATA_CREATION_VALIDATION_MAP_ = { + "video": validate_video_project_creation_data_impl_ +} + + +def validate_project_creation_data_(proj, project_id, type): + if type not in PROJECT_DATA_CREATION_VALIDATION_MAP_: + raise ValueError(f"Invalid project creation data type: {type}") + return PROJECT_DATA_CREATION_VALIDATION_MAP_[type](proj, project_id) + + +def labelbox_get_project_json_with_id_(client, project_id, fail_on_error=False): + # Set the export params to include/exclude certain fields. + export_params = { + "attachments": False, + "metadata_fields": False, + "data_row_details": True, + "project_details": True, + "label_details": True, + "performance_details": False, + "interpolated_frames": True, + "embeddings": False, + } + + project = client.get_project(project_id) + export_task = project.export_v2(params=export_params) + + print( + "requesting project info from labelbox with id", + project_id, + "export task id", + export_task.uid, + ) + export_task.wait_till_done() + + if export_task.errors: + if fail_on_error: + raise ValueError( + f"Labelbox export task failed with errors: {export_task.errors}" + ) + print("Labelbox export task failed with errors:", export_task.errors) + + print("project info is ready for project with id", project_id) + + return export_task.result + + +def create_tensors_default_(ds): + ds.create_tensor("frames", htype="image", sample_compression="jpg") + ds.create_tensor("frame_idx", htype="generic", dtype="int32") + ds.create_tensor("video_idx", htype="generic", dtype="int32") + + +def fill_data_default_(ds, group_ids, indexes, frames): + ds["frames"].extend(frames) + ds["video_idx"].extend(group_ids) + ds["frame_idx"].extend(indexes) diff --git a/deeplake/integrations/labelbox/v3_converters.py b/deeplake/integrations/labelbox/v3_converters.py new file mode 100644 index 0000000000..1e7b3b627c --- /dev/null +++ b/deeplake/integrations/labelbox/v3_converters.py @@ -0,0 +1,335 @@ +from PIL import Image +import urllib.request +import numpy as np +import copy + + +def bbox_converter_(obj, converter, tensor_name, context, generate_labels): + ds = context["ds"] + try: + ds.create_tensor( + tensor_name, + htype="bbox", + dtype="int32", + coords={"type": "pixel", "mode": "LTWH"}, + ) + except: + pass + + if generate_labels: + print("bbox converter does not support generating labels") + + converter.register_feature_id_for_kind("tool", "bounding_box", obj, tensor_name) + + def bbox_converter(row, obj): + if tensor_name not in converter.values_cache: + converter.values_cache[tensor_name] = dict() + if row not in converter.values_cache[tensor_name]: + converter.values_cache[tensor_name][row] = [] + + converter.values_cache[tensor_name][row].append( + [ + int(v) + for v in [ + obj["bounding_box"]["left"], + obj["bounding_box"]["top"], + obj["bounding_box"]["width"], + obj["bounding_box"]["height"], + ] + ] + ) + + converter.regsistered_actions[obj.feature_schema_id] = bbox_converter + + def interpolator(start, end, progress): + start_box = start["bounding_box"] + end_box = end["bounding_box"] + bbox = copy.deepcopy(start) + bbox["bounding_box"] = { + "top": start_box["top"] + (end_box["top"] - start_box["top"]) * progress, + "left": start_box["left"] + + (end_box["left"] - start_box["left"]) * progress, + "width": start_box["width"] + + (end_box["width"] - start_box["width"]) * progress, + "height": start_box["height"] + + (end_box["height"] - start_box["height"]) * progress, + } + + return bbox + + converter.registered_interpolators[obj.feature_schema_id] = interpolator + + +def radio_converter_(obj, converter, tensor_name, context, generate_labels): + ds = context["ds"] + + converter.label_mappings[tensor_name] = { + options.value: i for i, options in enumerate(obj.options) + } + + if generate_labels: + print("radio converter does not support generating labels") + + try: + ds.create_tensor( + tensor_name, + htype="class_label", + class_names=list(converter.label_mappings[tensor_name].keys()), + chunk_compression="lz4", + ) + except: + pass + + converter.register_feature_id_for_kind( + "annotation", "radio_answer", obj, tensor_name + ) + + def radio_converter(row, o): + if tensor_name not in converter.values_cache: + converter.values_cache[tensor_name] = dict() + if row not in converter.values_cache[tensor_name]: + converter.values_cache[tensor_name][row] = [] + converter.values_cache[tensor_name][row] = [ + converter.label_mappings[tensor_name][o["value"]] + ] + + for option in obj.options: + converter.regsistered_actions[option.feature_schema_id] = radio_converter + + def radio_converter_nested(row, obj): + radio_converter(row, obj["radio_answer"]) + + converter.regsistered_actions[obj.feature_schema_id] = radio_converter_nested + + +def checkbox_converter_(obj, converter, tensor_name, context, generate_labels): + ds = context["ds"] + + converter.label_mappings[tensor_name] = { + options.value: i for i, options in enumerate(obj.options) + } + + if generate_labels: + print("checkbox converter does not support generating labels") + + try: + ds.create_tensor( + tensor_name, + htype="class_label", + class_names=list(converter.label_mappings[tensor_name].keys()), + chunk_compression="lz4", + ) + except: + pass + + converter.register_feature_id_for_kind( + "annotation", "checklist_answers", obj, tensor_name + ) + + def checkbox_converter(row, obj): + if tensor_name not in converter.values_cache: + converter.values_cache[tensor_name] = dict() + if row not in converter.values_cache[tensor_name]: + converter.values_cache[tensor_name][row] = [] + + converter.values_cache[tensor_name][row].append( + converter.label_mappings[tensor_name][obj["value"]] + ) + + for option in obj.options: + converter.regsistered_actions[option.feature_schema_id] = checkbox_converter + + def checkbox_converter_nested(row, obj): + for o in obj["checklist_answers"]: + checkbox_converter(row, o) + + converter.regsistered_actions[obj.feature_schema_id] = checkbox_converter_nested + + +def point_converter_(obj, converter, tensor_name, context, generate_labels): + ds = context["ds"] + try: + ds.create_tensor(tensor_name, htype="point", dtype="int32") + except: + pass + + converter.register_feature_id_for_kind("annotation", "point", obj, tensor_name) + + if generate_labels: + print("point converter does not support generating labels") + + def point_converter(row, obj): + if tensor_name not in converter.values_cache: + converter.values_cache[tensor_name] = dict() + if row not in converter.values_cache[tensor_name]: + converter.values_cache[tensor_name][row] = [] + + converter.values_cache[tensor_name][row].append( + [int(obj["point"]["x"]), int(obj["point"]["y"])] + ) + + converter.regsistered_actions[obj.feature_schema_id] = point_converter + + def interpolator(start, end, progress): + start_point = start["point"] + end_point = end["point"] + point = copy.deepcopy(start) + point["point"] = { + "x": start_point["x"] + (end_point["x"] - start_point["x"]) * progress, + "y": start_point["y"] + (end_point["y"] - start_point["y"]) * progress, + } + + return point + + converter.registered_interpolators[obj.feature_schema_id] = interpolator + + +def line_converter_(obj, converter, tensor_name, context, generate_labels): + ds = context["ds"] + try: + ds.create_tensor(tensor_name, htype="polygon", dtype="int32") + except: + pass + + converter.register_feature_id_for_kind("annotation", "line", obj, tensor_name) + + if generate_labels: + print("line converter does not support generating labels") + + def polygon_converter(row, obj): + if tensor_name not in converter.values_cache: + converter.values_cache[tensor_name] = dict() + if row not in converter.values_cache[tensor_name]: + converter.values_cache[tensor_name][row] = [] + + converter.values_cache[tensor_name][row].append( + [[int(l["x"]), int(l["y"])] for l in obj["line"]] + ) + + converter.regsistered_actions[obj.feature_schema_id] = polygon_converter + + def interpolator(start, end, progress): + start_line = start["line"] + end_line = end["line"] + line = copy.deepcopy(start) + line["line"] = [ + [ + start_line[i]["x"] + (end_line[i]["x"] - start_line[i]["x"]) * progress, + start_line[i]["y"] + (end_line[i]["y"] - start_line[i]["y"]) * progress, + ] + for i in range(len(start_line)) + ] + + return line + + converter.registered_interpolators[obj.feature_schema_id] = interpolator + + +def raster_segmentation_converter_( + obj, converter, tensor_name, context, generate_labels +): + ds = context["ds"] + try: + ds.create_tensor( + tensor_name, htype="binary_mask", dtype="bool", sample_compression="lz4" + ) + except: + pass + + try: + if generate_labels: + ds.create_tensor( + f"{tensor_name}_labels", + htype="class_label", + dtype="int32", + class_names=[], + chunk_compression="lz4", + ) + converter.label_mappings[f"{tensor_name}_labels"] = dict() + except: + pass + + converter.register_feature_id_for_kind( + "annotation", "raster-segmentation", obj, tensor_name + ) + + tool_name = obj.name + + def mask_converter(row, obj): + try: + r = urllib.request.Request( + obj["mask"]["url"], + headers={"Authorization": f'Bearer {context["lb_api_key"]}'}, + ) + with urllib.request.urlopen(r) as response: + if generate_labels: + if ( + tool_name + not in converter.label_mappings[f"{tensor_name}_labels"] + ): + converter.label_mappings[f"{tensor_name}_labels"][tool_name] = ( + len(converter.label_mappings[f"{tensor_name}_labels"]) + ) + ds[f"{tensor_name}_labels"].info.update( + class_names=list( + converter.label_mappings[f"{tensor_name}_labels"].keys() + ) + ) + val = [] + try: + val = ( + ds[f"{tensor_name}_labels"][row].numpy(aslist=True).tolist() + ) + except (KeyError, IndexError): + pass + + val.append( + converter.label_mappings[f"{tensor_name}_labels"][tool_name] + ) + ds[f"{tensor_name}_labels"][row] = val + + mask = np.array(Image.open(response)).astype(np.bool_) + mask = mask[..., np.newaxis] + try: + if generate_labels: + val = ds[tensor_name][row].numpy() + labels = ds[f"{tensor_name}_labels"].info["class_names"] + if len(labels) != val.shape[-1]: + val = np.concatenate( + [val, np.zeros_like(mask)], + axis=-1, + ) + idx = labels.index(tool_name) + val[:, :, idx] = np.logical_or(val[:, :, idx], mask[:, :, 0]) + else: + val = np.logical_or(ds[tensor_name][row].numpy(), mask) + except (KeyError, IndexError): + val = mask + + ds[tensor_name][row] = val + except Exception as e: + print(f"Error downloading mask: {e}") + + converter.regsistered_actions[obj.feature_schema_id] = mask_converter + + +def text_converter_(obj, converter, tensor_name, context, generate_labels): + ds = context["ds"] + try: + ds.create_tensor(tensor_name, htype="text", dtype="str") + except: + pass + + converter.register_feature_id_for_kind("annotation", "text", obj, tensor_name) + + if generate_labels: + print("text converter does not support generating labels") + + def text_converter(row, obj): + if tensor_name not in converter.values_cache: + converter.values_cache[tensor_name] = dict() + if row not in converter.values_cache[tensor_name]: + converter.values_cache[tensor_name][row] = [] + converter.values_cache[tensor_name][row] = obj["text_answer"]["content"] + + converter.regsistered_actions[obj.feature_schema_id] = text_converter diff --git a/deeplake/integrations/tests/test_labelbox.py b/deeplake/integrations/tests/test_labelbox.py new file mode 100644 index 0000000000..2d5010a755 --- /dev/null +++ b/deeplake/integrations/tests/test_labelbox.py @@ -0,0 +1,251 @@ +import labelbox as lb # type: ignore +import os +import numpy as np +import pytest + +from deeplake.integrations.labelbox import ( + create_dataset_from_video_annotation_project, + converter_for_video_project_with_id, + load_blob_file_paths_from_azure, +) + + +def validate_ds(ds): + assert set(ds.tensors) == set( + { + "bbox/bbox", + "bbox/fully_visible", + "checklist", + "frame_idx", + "frames", + "line", + "mask/mask", + "mask/mask_label", + "mask/mask_labels", + "metadata/current_frame_name", + "metadata/data_row_id", + "metadata/dataset_id", + "metadata/dataset_name", + "metadata/frame_count", + "metadata/frame_number", + "metadata/frame_rate", + "metadata/global_key", + "metadata/height", + "metadata/label_creator", + "metadata/name", + "metadata/ontology_id", + "metadata/project_name", + "metadata/width", + "point", + "radio_bttn", + "radio_bttn_scale", + "text", + "video_idx", + } + ) + + assert ds.max_len == 876 + + assert len(ds["radio_bttn"]) == 474 + assert np.all(ds["radio_bttn"][0].numpy() == [[0]]) + assert np.all(ds["radio_bttn"][20].numpy() == [[0]]) + assert np.all(ds["radio_bttn"][23].numpy() == [[1]]) + + assert np.all( + ds["bbox/bbox"][0:3].numpy() + == [[[1092, 9, 361, 361]], [[1092, 8, 360, 361]], [[1093, 8, 361, 360]]] + ) + assert np.all(ds["bbox/fully_visible"][0:3].numpy() == [[0], [0], [0]]) + + assert np.all(ds["bbox/bbox"][499].numpy() == [[1463, 0, 287, 79]]) + assert len(ds["bbox/bbox"]) == 500 + + assert np.all(ds["bbox/fully_visible"][499].numpy() == [[1]]) + assert len(ds["bbox/fully_visible"]) == 500 + + assert np.all(ds["radio_bttn"][0].numpy() == [[0]]) + assert np.all(ds["radio_bttn"][0].numpy() == [[0]]) + + assert np.all(ds["checklist"][499].numpy() == [[]]) + assert np.all(ds["checklist"][500].numpy() == [[0, 1]]) + assert np.all(ds["checklist"][598].numpy() == [[1, 0]]) + assert np.all(ds["checklist"][599].numpy() == [[0]]) + assert np.all(ds["checklist"][698].numpy() == [[0]]) + assert np.all(ds["checklist"][699].numpy() == [[1]]) + assert len(ds["checklist"]) == 739 + + assert np.all( + ds["frame_idx"][245:255].numpy() + == [[245], [246], [247], [248], [249], [250], [251], [252], [253], [254]] + ) + + assert np.all( + ds["frame_idx"][495:505].numpy() + == [[495], [496], [497], [498], [499], [0], [1], [2], [3], [4]] + ) + + assert np.all(ds["line"][245:255].numpy() == []) + + assert np.all(ds["mask/mask_label"][500].numpy() == [1]) + assert np.all(ds["mask/mask_label"][739].numpy() == [0]) + + assert np.all(ds["mask/mask_labels"][500].numpy() == [0, 1]) + assert np.all(ds["mask/mask_labels"][739].numpy() == [0]) + + assert np.all( + ds["metadata/current_frame_name"][245:255].numpy() + == [ + ["output005_000245"], + ["output005_000246"], + ["output005_000247"], + ["output005_000248"], + ["output005_000249"], + ["output005_000250"], + ["output005_000251"], + ["output005_000252"], + ["output005_000253"], + ["output005_000254"], + ] + ) + + assert np.all( + ds["metadata/current_frame_name"][495:505].numpy() + == [ + ["output005_000495"], + ["output005_000496"], + ["output005_000497"], + ["output005_000498"], + ["output005_000499"], + ["output004_000000"], + ["output004_000001"], + ["output004_000002"], + ["output004_000003"], + ["output004_000004"], + ] + ) + + assert np.all( + ds["video_idx"][245:255].numpy() + == [[0], [0], [0], [0], [0], [0], [0], [0], [0], [0]] + ) + + assert np.all( + ds["video_idx"][495:505].numpy() + == [[0], [0], [0], [0], [0], [1], [1], [1], [1], [1]] + ) + + assert len(ds["point"]) == 857 + assert np.all(ds["point"][0].numpy() == [[]]) + assert np.all(ds["point"][499].numpy() == [[]]) + assert np.all(ds["point"][800].numpy() == [[1630, 49]]) + + print("dataset is valid!") + + +def get_azure_sas_token(): + import datetime + + from azure.identity import DefaultAzureCredential + from azure.storage.blob import ( + BlobServiceClient, + ContainerSasPermissions, + generate_container_sas, + ) + + # Construct the blob endpoint from the account name + account_url = "https://activeloopgen2.blob.core.windows.net" + + # Create a BlobServiceClient object using DefaultAzureCredential + blob_service_client = BlobServiceClient( + account_url, credential=DefaultAzureCredential() + ) + # Get a user delegation key that's valid for 1 day + delegation_key_start_time = datetime.datetime.now(datetime.timezone.utc) + delegation_key_expiry_time = delegation_key_start_time + datetime.timedelta(days=1) + + user_delegation_key = blob_service_client.get_user_delegation_key( + key_start_time=delegation_key_start_time, + key_expiry_time=delegation_key_expiry_time, + ) + + start_time = datetime.datetime.now(datetime.timezone.utc) + expiry_time = start_time + datetime.timedelta(days=1) + + sas_token = generate_container_sas( + account_name="activeloopgen2", + container_name="deeplake-tests", + user_delegation_key=user_delegation_key, + permission=ContainerSasPermissions(read=True, list=True), + expiry=expiry_time, + start=start_time, + ) + + return sas_token + + +@pytest.mark.skip(reason="labelbox api sometimes freezes") +def test_connect_to_labelbox(): + # the path where we want to create the dataset + ds_path = "mem://labelbox_connect_test" + + API_KEY = os.environ["LABELBOX_TOKEN"] + client = lb.Client(api_key=API_KEY) + + project_id = "cm4hts5gf0109072nbpl390xc" + + sas_token = get_azure_sas_token() + + # we pass the url presigner in cases when the videos are in cloud storage ( + # for this case azure blob storage) and the videos were added to labelbox with their integrations functionality. + # the default one tries to use labelbox api to get the non public remote urls. + def url_presigner(url): + # the second value is the headers that will be added to the request + return url.partition("?")[0] + "?" + sas_token, {} + + ds = create_dataset_from_video_annotation_project( + ds_path, + project_id, + client, + API_KEY, + url_presigner=url_presigner, + ) + + def ds_provider(p): + # we need to have a clean branch to apply the annotations + try: + ds.delete_branch("labelbox") + except: + pass + ds.checkout("labelbox", create=True) + return ds + + converter = converter_for_video_project_with_id( + project_id, + client, + ds_provider, + API_KEY, + group_mapping={"raster-segmentation": "mask"}, + ) + print("generating annotations") + ds = converter.dataset_with_applied_annotations() + + # commit the annotations to the dataset + ds.commit("add labelbox annotations") + + validate_ds(ds) + + +@pytest.mark.skip(reason="somemtimes fails with timeout") +def test_labelbox_azure_utils(): + files = load_blob_file_paths_from_azure( + "activeloopgen2", + "deeplake-tests", + "video_chunks", + get_azure_sas_token(), + lambda x: x.endswith(".mp4"), + ) + assert set([os.path.basename(f.partition("?")[0]) for f in files]) == { + "output004.mp4", + "output005.mp4", + "output006.mp4", + } diff --git a/deeplake/requirements/common.txt b/deeplake/requirements/common.txt index 942d0ad053..10e8bc2280 100644 --- a/deeplake/requirements/common.txt +++ b/deeplake/requirements/common.txt @@ -25,3 +25,4 @@ azure-identity azure-storage-blob pydantic numpy-stl +labelbox diff --git a/pyproject.toml b/pyproject.toml index 4ba342d826..bd6436afc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ google-auth = { version = "~2.0.1", optional = true } google-auth-oauthlib = { version = "~0.4.5", optional = true } google-api-python-client = { version = "~2.31.0", optional = true } oauth2client = { version = "~4.1.3", optional = true } +labelbox = { optional = true } [tool.poetry.extras] audio = ["av"] @@ -57,6 +58,7 @@ gdrive = [ point_cloud = ["laspy"] mesh = ["laspy", "numpy-stl"] enterprise = ["pyjwt"] +labelbox = ["labelbox", "av", "pillow"] all = [ "av", "google-cloud-storage", @@ -74,6 +76,7 @@ all = [ "laspy", "numpy-stl", "pyjwt", + "labelbox", ] diff --git a/setup.py b/setup.py index d72e5b9a93..dcd32e4415 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ ], "point_cloud": ["laspy"], "mesh": ["laspy", "numpy-stl"], + "labelbox": ["labelbox", "av", "pillow"], }