Merge branch 'main' into multiprocessed-camera

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ""
+labels: bug
+assignees: ""
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Environment:**
+
+- Python:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,10 @@
+---
+name: Documentation
+about: Suggest a change to the documentation
+title: ""
+labels: documentation
+assignees: ""
+---
+
+**Documentation**
+Describe where you believe documentation is missing or should be improved.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,16 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ""
+labels: enhancement
+assignees: ""
+---
+
+**Describe the feature you'd like**
+A clear and concise description of what you'd like to have in airo-mono.
+
+**Use cases**
+Describe for who and when your feature would be useful.
+
+**Possible implementation**
+Describe any ideas you have for how this could work.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,10 @@
+---
+name: Question
+about: Ask a question about airo-mono
+title: ""
+labels: question
+assignees: ""
+---
+
+**Question**
+Ask your question here.

airo-camera-toolkit/README.md

@@ -88,7 +88,7 @@ See [reprojection.py](./airo_camera_toolkit/reprojection.py) for more details.
 
 ## Annotation tool
 
-See [annotation_tool.md](annotation_tool.md) for usage instructions.
+See [annotation_tool.md](./airo_camera_toolkit/annotation_tool.md) for usage instructions.
 
 ## Image Transforms
 

airo-camera-toolkit/airo_camera_toolkit/calibration/fiducial_markers.py

@@ -9,7 +9,6 @@
 
 import cv2
 import numpy as np
-from airo_camera_toolkit.reprojection import project_frame_to_image_plane
 from airo_spatial_algebra import SE3Container
 from airo_typing import CameraIntrinsicsMatrixType, HomogeneousMatrixType, OpenCVIntImageType
 from cv2 import aruco
@@ -144,6 +143,38 @@ def get_pose_of_charuco_board(
     return charuco_pose_in_camera_frame
 
 
+def detect_charuco_board(
+    image: OpenCVIntImageType,
+    camera_matrix: CameraIntrinsicsMatrixType,
+    dist_coeffs: Optional[np.ndarray] = None,
+    aruco_markers: ArucoDictType = AIRO_DEFAULT_ARUCO_DICT,
+    charuco_board: CharucoDictType = AIRO_DEFAULT_CHARUCO_BOARD,
+) -> Optional[HomogeneousMatrixType]:
+    """Detect the pose of a charuco board from an image and the camera's intrinsics.
+
+    Args:
+        image: An image that might contain a charuco board.
+        camera_matrix: The intrinsics of the camera that took the image.
+        dist_coeffs: The distortion coefficients of the camera that took the image.
+        aruco_markers: The dictionary from OpenCV that specifies the aruco marker parameters.
+        charuco_board: The dictionary from OpenCV that specifies the charuco board parameters.
+
+    Returns:
+        Optional[HomogeneousMatrixType]: The pose of the charuco board in the camera frame, if it was detected.
+    """
+
+    aruco_result = detect_aruco_markers(image, aruco_markers)
+    if not aruco_result:
+        return None
+
+    charuco_result = detect_charuco_corners(image, aruco_result, charuco_board)
+    if not charuco_result:
+        return None
+
+    charuco_pose = get_pose_of_charuco_board(charuco_result, charuco_board, camera_matrix, dist_coeffs)
+    return charuco_pose
+
+
 #################
 # visualization #
 #################
@@ -152,27 +183,24 @@ def get_pose_of_charuco_board(
 def draw_frame_on_image(
     image: OpenCVIntImageType, frame_pose_in_camera: HomogeneousMatrixType, camera_matrix: CameraIntrinsicsMatrixType
 ) -> OpenCVIntImageType:
-    """draws a 2D projection of a frame on the iamge. Be careful when interpreting this visually, it is often hard to estimate the true 3D direction of an axis' 2D projection."""
-    project_points = np.array([[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
-    origin, x_pos, y_pos, z_pos = project_frame_to_image_plane(
-        project_points, camera_matrix, frame_pose_in_camera
-    ).astype(int)
-    image = cv2.line(image, x_pos, origin, color=(0, 0, 255), thickness=2)
-    image = cv2.line(image, y_pos, origin, color=(0, 255, 0), thickness=2)
-    image = cv2.line(image, z_pos, origin, color=(255, 0, 0), thickness=2)
+    """Draws a 2D projection of a frame on the image. Be careful when interpreting this visually, it is often hard to estimate the true 3D direction of an axis' 2D projection."""
+    charuco_se3 = SE3Container.from_homogeneous_matrix(frame_pose_in_camera)
+    rvec = charuco_se3.orientation_as_rotation_vector
+    tvec = charuco_se3.translation
+    image = cv2.drawFrameAxes(image, camera_matrix, None, rvec, tvec, 0.2)
     return image
 
 
 def visualize_aruco_detections(
     image: OpenCVIntImageType, aruco_result: ArucoMarkerDetectionResult
 ) -> OpenCVIntImageType:
-    """draws the aruco marker countours/corners and their IDs on the image"""
+    """Draws the aruco marker countours/corners and their IDs on the image"""
     image = aruco.drawDetectedMarkers(image, [x for x in aruco_result.corners], aruco_result.ids)
     return image
 
 
 def visualize_charuco_detection(image: OpenCVIntImageType, result: CharucoCornerDetectionResult) -> OpenCVIntImageType:
-    """draws the charuco checkerboard corners and their IDs on the image"""
+    """Draws the charuco checkerboard corners and their IDs on the image"""
     image = aruco.drawDetectedCornersCharuco(image, np.array(result.corners), np.array(result.ids), (255, 255, 0))
     return image
 
@@ -197,7 +225,6 @@ def visualize_marker_detections(
         charuco_y_count: Optional[int] = None,
         charuco_tile_size: Optional[int] = None,
     ) -> None:
-
         aruco_dict = AIRO_DEFAULT_ARUCO_DICT
         detect_charuco = charuco_x_count is not None and charuco_y_count is not None and charuco_tile_size is not None
         if detect_charuco:

airo-camera-toolkit/docs/live_charuco_pose.py

@@ -0,0 +1,21 @@
+import cv2
+from airo_camera_toolkit.calibration.fiducial_markers import detect_charuco_board, draw_frame_on_image
+from airo_camera_toolkit.cameras.zed2i import Zed2i
+from airo_camera_toolkit.utils import ImageConverter
+
+camera = Zed2i(fps=30)
+intrinsics = camera.intrinsics_matrix()
+
+window_name = "Charuco Pose"
+cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
+
+while True:
+    image = camera.get_rgb_image_as_int()
+    image = ImageConverter.from_numpy_int_format(image).image_in_opencv_format
+    pose = detect_charuco_board(image, intrinsics)
+    if pose is not None:
+        draw_frame_on_image(image, pose, intrinsics)
+    cv2.imshow(window_name, image)
+    key = cv2.waitKey(1)
+    if key == ord("q"):
+        break

airo-dataset-tools/airo_dataset_tools/cli.py

@@ -4,13 +4,12 @@
 import os
 from typing import List, Optional
 
-import albumentations as A
 import click
 from airo_dataset_tools.coco_tools.coco_instances_to_yolo import create_yolo_dataset_from_coco_instances_dataset
-from airo_dataset_tools.coco_tools.transform_dataset import apply_transform_to_coco_dataset
+from airo_dataset_tools.coco_tools.fiftyone_viewer import view_coco_dataset
+from airo_dataset_tools.coco_tools.split_dataset import split_and_save_coco_dataset
+from airo_dataset_tools.coco_tools.transform_dataset import resize_coco_keypoints_dataset
 from airo_dataset_tools.cvat_labeling.convert_cvat_to_coco import cvat_image_to_coco
-from airo_dataset_tools.data_parsers.coco import CocoKeypointsDataset
-from airo_dataset_tools.fiftyone_viewer import view_coco_dataset
 
 
 @click.group()
@@ -42,11 +41,16 @@ def view_coco_dataset_cli(
 
 @cli.command(name="convert-cvat-to-coco-keypoints")
 @click.argument("cvat_xml_file", type=str, required=True)
+@click.argument("coco_categories_json_file", type=str, required=True)
 @click.option("--add_bbox", is_flag=True, default=False, help="include bounding box in coco annotations")
 @click.option("--add_segmentation", is_flag=True, default=False, help="include segmentation in coco annotations")
-def convert_cvat_to_coco_cli(cvat_xml_file: str, add_bbox: bool, add_segmentation: bool) -> None:
-    """Convert CVAT XML to COCO keypoints json"""
-    coco = cvat_image_to_coco(cvat_xml_file, add_bbox=add_bbox, add_segmentation=add_segmentation)
+def convert_cvat_to_coco_cli(
+    cvat_xml_file: str, coco_categories_json_file: str, add_bbox: bool, add_segmentation: bool
+) -> None:
+    """Convert CVAT XML to COCO keypoints json according to specified coco categories"""
+    coco = cvat_image_to_coco(
+        cvat_xml_file, coco_categories_json_file, add_bbox=add_bbox, add_segmentation=add_segmentation
+    )
     path = os.path.dirname(cvat_xml_file)
     filename = os.path.basename(cvat_xml_file)
     path = os.path.join(path, filename.split(".")[0] + ".json")
@@ -58,38 +62,39 @@ def convert_cvat_to_coco_cli(cvat_xml_file: str, add_bbox: bool, add_segmentatio
 @click.argument("annotations-json-path", type=click.Path(exists=True))
 @click.option("--width", type=int, required=True)
 @click.option("--height", type=int, required=True)
-def resize_coco_keypoints_dataset(annotations_json_path: str, width: int, height: int) -> None:
+def resize_coco_keypoints_dataset_cli(annotations_json_path: str, width: int, height: int) -> None:
     """Resize a COCO dataset. Will create a new directory with the resized dataset on the same level as the original dataset.
     Dataset is assumed to be
     /dir
         annotations.json # contains relative paths w.r.t. /dir
         ...
     """
-    coco_dataset_dir = os.path.dirname(annotations_json_path)
-    annotations_file_name = os.path.basename(annotations_json_path)
-    dataset_parent_dir = os.path.dirname(coco_dataset_dir)
-    transformed_dataset_dir = os.path.join(
-        dataset_parent_dir, f"{annotations_file_name.split('.')[0]}_resized_{width}x{height}"
-    )
-    os.makedirs(transformed_dataset_dir, exist_ok=True)
-
-    transforms = [A.Resize(height, width)]
-    coco_json = json.load(open(annotations_json_path, "r"))
-    coco_dataset = CocoKeypointsDataset(**coco_json)
-    transformed_dataset = apply_transform_to_coco_dataset(
-        transforms, coco_dataset, coco_dataset_dir, transformed_dataset_dir
-    )
-
-    transformed_dataset_dict = transformed_dataset.dict(exclude_none=True)
-    with open(os.path.join(transformed_dataset_dir, annotations_file_name), "w") as f:
-        json.dump(transformed_dataset_dict, f)
+    resize_coco_keypoints_dataset(annotations_json_path, width, height)
 
 
-@click.command(name="coco-instances-to-yolo")
+@cli.command(name="coco-instances-to-yolo")
 @click.option("--coco_json", type=str)
 @click.option("--target_dir", type=str)
 @click.option("--use_segmentation", is_flag=True)
 def coco_intances_to_yolo(coco_json: str, target_dir: str, use_segmentation: bool) -> None:
     """Create a YOLO detections/segmentations dataset from a coco instances dataset"""
     print(f"converting coco instances dataset {coco_json} to yolo dataset {target_dir}")
     create_yolo_dataset_from_coco_instances_dataset(coco_json, target_dir, use_segmentation=use_segmentation)
+
+
+@cli.command(name="split-coco-dataset")
+@click.argument("json-path", type=click.Path(exists=True))
+@click.option("--split-ratios", type=float, multiple=True, required=True)
+@click.option("--shuffle-before-splitting", is_flag=True, default=True)
+def split_coco_dataset_cli(json_path: str, split_ratios: List[float], shuffle_before_splitting: bool) -> None:
+    """Split a COCO dataset into subsets according to the specified relative ratios and save them to disk.
+    Images are split with their corresponding annotations. No guarantees on class balance or annotation ratios.
+
+    If two ratios are specified, the dataset will be split into two subsets. these will be called train/val by default.
+    If three ratios are specified, the dataset will be split into three subsets. these will be called train/val/test by default.
+    """
+    split_and_save_coco_dataset(json_path, split_ratios, shuffle_before_splitting=shuffle_before_splitting)
+
+
+if __name__ == "__main__":
+    cli()

airo-dataset-tools/airo_dataset_tools/coco_tools/albumentations.py

@@ -0,0 +1,37 @@
+from typing import Any
+
+import albumentations as A
+import numpy as np
+from PIL import Image
+
+
+class PillowResize(A.Resize):  # type: ignore
+    """Use Pillow (instead of OpenCV) to resize the input to the given height and width.
+    always uses Bicubic interpolation.
+
+    PIllow instead of Opencv because opencv does not adapt the filter size to the scaling factor,
+    which creates artifacts in the output for large downscaling factors.
+    cf. https://arxiv.org/pdf/2104.11222.pdf
+
+    Args:
+        height (int): desired height of the output.
+        width (int): desired width of the output.
+        p (float): probability of applying the transform. Default: 1.
+
+    Targets:
+        image, mask, bboxes, keypoints
+
+    Image types:
+        uint8, float32
+    """
+
+    def __init__(
+        self, height: int, width: int, interpolation: Any = Image.BICUBIC, always_apply: bool = False, p: float = 1.0
+    ):
+        super(PillowResize, self).__init__(height, width, always_apply=always_apply, p=p)
+        self.height = height
+        self.width = width
+        self.interpolation = interpolation
+
+    def apply(self, img: np.ndarray, **params: dict) -> np.ndarray:
+        return np.array(Image.fromarray(img).resize((self.width, self.height), self.interpolation))

airo-dataset-tools/airo_dataset_tools/coco_tools/coco_instances_to_yolo.py

@@ -73,6 +73,13 @@ def create_yolo_dataset_from_coco_instances_dataset(
 
         relative_image_path = image_path.relative_to(_coco_dataset_json_path.parent)
 
+        # ultralytics parser finds the 'latest' occurance of 'images' in the dataset path,
+        #  so we need to replace any occurance of 'image' with 'img'
+        #  https://github.com/ultralytics/ultralytics/issues/3581
+
+        relative_image_path_str = str(relative_image_path).replace("image", "img")
+        relative_image_path = pathlib.Path(relative_image_path_str)
+
         image = cv2.imread(str(image_path))
         height, width, _ = image.shape
 
@@ -124,8 +131,7 @@ def create_yolo_dataset_from_coco_instances_dataset(
 
 
 if __name__ == "__main__":
-    """example usage of the above function to resize all images in a coco dataset.
-    Copy the following lines into your own codebase and modify as needed."""
+    """example usage"""
     import os
 
     path = pathlib.Path(__file__).parents[1] / "cvat_labeling" / "example" / "coco.json"