From 33a5c7c39caae074c49cf3cba1ba29bbbc690524 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Thu, 22 Feb 2024 18:35:49 +0100 Subject: [PATCH 01/13] OpenCVVideoCapture implementation (#130) --- CHANGELOG.md | 2 +- .../airo_camera_toolkit/cameras/README.md | 2 + .../cameras/opencv_videocapture/__init__.py | 0 .../opencv_videocapture.py | 115 ++++++++++++++++++ .../opencv_videocapture_camera.md | 6 + 5 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/__init__.py create mode 100644 airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py create mode 100644 airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture_camera.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 634070dc..bbb9f312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly - Functions to convert from our numpy-based dataclass to and from open3d point clouds - `BoundingBox3DType` - `Zed2i.ULTRA_DEPTH_MODE` to enable the ultra depth setting for the Zed2i cameras - +- `OpenCVVideoCapture` implementation of `RGBCamera` for working with arbitrary cameras ### Changed diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md b/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md index db373564..a09afe19 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md @@ -7,6 +7,8 @@ This subpackage contains implementations of the camera interface for the cameras It also contains code to enable multiprocessed use of the camera streams: [multiprocessed camera](./multiprocess/) +There is also an implementation for generic RGB cameras using OpenCV `VideoCapture`: [OpenCV VideoCapture](./opencv_videocapture/) + ## 1. Installation Implementations usually require the installation of SDKs, drivers etc. to communicate with the camera. This information can be found in `READMEs` for each camera: diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/__init__.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py new file mode 100644 index 00000000..af09b3e7 --- /dev/null +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import math +import os +from typing import Any, Optional, Tuple + +import cv2 +from airo_camera_toolkit.interfaces import RGBCamera +from airo_camera_toolkit.utils.image_converter import ImageConverter +from airo_typing import CameraIntrinsicsMatrixType, CameraResolutionType, NumpyFloatImageType, NumpyIntImageType + + +class OpenCVVideoCapture(RGBCamera): + """Wrapper around OpenCV's VideoCapture so we can test the camera interface without external cameras.""" + + # Some standard resolutions that are likely to be supported by webcams. + # 16:9 + RESOLUTION_1080 = (1920, 1080) + RESOLUTION_720 = (1280, 720) + # 4:3 + RESOLUTION_768 = (1024, 768) + RESOLUTION_480 = (640, 480) + + def __init__( + self, + video_capture_args: Tuple[Any] = (0,), + intrinsics_matrix: Optional[CameraIntrinsicsMatrixType] = None, + resolution: CameraResolutionType = RESOLUTION_480, + fps: int = 30, + ) -> None: + self.video_capture = cv2.VideoCapture(*video_capture_args) + + # If passing a video file, we want to check if it exists. Then, we can throw a more meaningful + # error if it does not. + if len(video_capture_args) > 0 and isinstance(video_capture_args[0], str): + if not os.path.isfile(video_capture_args[0]): + raise FileNotFoundError(f"Could not find video file {video_capture_args[0]}") + if not self.video_capture.isOpened(): + raise RuntimeError(f"Cannot open camera {video_capture_args[0]}. Is it connected?") + + # Note that the following will not forcibly set the resolution. If the user's webcam + # does not support the desired resolution, OpenCV will silently select a close match. + self.video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, resolution[0]) + self.video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, resolution[1]) + self.video_capture.set(cv2.CAP_PROP_FPS, fps) + + self._intrinsics_matrix = intrinsics_matrix + + self.fps = self.video_capture.get(cv2.CAP_PROP_FPS) + self._resolution = ( + math.floor(self.video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)), + math.floor(self.video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + + @property + def resolution(self) -> CameraResolutionType: + return self._resolution + + def __enter__(self) -> RGBCamera: + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.video_capture.release() + + def intrinsics_matrix(self) -> CameraIntrinsicsMatrixType: + """Obtain the intrinsics matrix of the camera. + + Raises: + RuntimeError: You must explicitly pass an intrinsics object to the constructor. + + Returns: + CameraIntrinsicsMatrixType: The intrinsics matrix. + """ + if self._intrinsics_matrix is None: + raise RuntimeError( + "OpenCVVideoCapture does not have a preset intrinsics matrix. Pass it to the constructor if you know it." + ) + return self._intrinsics_matrix + + def _grab_images(self) -> None: + ret, image = self.video_capture.read() + if not ret: # When streaming a video, we will at some point reach the end. + raise EOFError("Can't receive frame (stream end?). Exiting...") + + self._frame = image + + def _retrieve_rgb_image(self) -> NumpyFloatImageType: + return ImageConverter.from_opencv_format(self._frame).image_in_numpy_format + + def _retrieve_rgb_image_as_int(self) -> NumpyIntImageType: + return ImageConverter.from_opencv_format(self._frame).image_in_numpy_int_format + + +if __name__ == "__main__": + import airo_camera_toolkit.cameras.manual_test_hw as test + import numpy as np + + camera = OpenCVVideoCapture(intrinsics_matrix=np.eye(3)) + + # Perform tests + test.manual_test_camera(camera) + test.manual_test_rgb_camera(camera) + test.profile_rgb_throughput(camera) + + # Live viewer + cv2.namedWindow("OpenCV Webcam RGB", cv2.WINDOW_NORMAL) + + while True: + color_image = camera.get_rgb_image_as_int() + color_image = ImageConverter.from_numpy_int_format(color_image).image_in_opencv_format + + cv2.imshow("OpenCV Webcam RGB", color_image) + key = cv2.waitKey(1) + if key == ord("q"): + break diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture_camera.md b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture_camera.md new file mode 100644 index 00000000..1f7e773d --- /dev/null +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture_camera.md @@ -0,0 +1,6 @@ +# Generic OpenCV camera + +This `RGBCamera` implementation allows testing arbitrary cameras through the OpenCV `VideoCapture` interface. + +We currently do not support intrinsics calibration in airo-camera-toolkit. You can find the intrinsics of your camera +using [these instructions](https://docs.opencv.org/4.x/dc/dbb/tutorial_py_calibration.html). From d46a94776914b8c1eeeaef633d8810f1cf8aacf5 Mon Sep 17 00:00:00 2001 From: victorlouisdg Date: Tue, 27 Feb 2024 13:04:26 +0100 Subject: [PATCH 02/13] make airo-robots optional dep of airo-camera-toolkit --- airo-camera-toolkit/README.md | 5 +++++ airo-camera-toolkit/setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/airo-camera-toolkit/README.md b/airo-camera-toolkit/README.md index e577b8b8..c4328a24 100644 --- a/airo-camera-toolkit/README.md +++ b/airo-camera-toolkit/README.md @@ -26,6 +26,11 @@ Instructions can be found in the following files: * [ZED Installation](airo_camera_toolkit/cameras/zed_installation.md) * [RealSense Installation](airo_camera_toolkit/cameras/realsense_installation.md) +Additionally, to ensure you have `airo-robots` installed for the hand-eye calibration, install the extra dependencies: +``` +pip install .[hand-eye-calibration] +``` + ## Getting started with cameras Camera can be accessed by instantiating the corresponding class:, e.g. for a ZED camera: ```python diff --git a/airo-camera-toolkit/setup.py b/airo-camera-toolkit/setup.py index 23e2b7f1..1df084e7 100644 --- a/airo-camera-toolkit/setup.py +++ b/airo-camera-toolkit/setup.py @@ -20,9 +20,9 @@ "loguru", "airo-typing", "airo-spatial-algebra", - "airo-robots", "airo-dataset-tools", ], + extras_require={"hand-eye-calibration": ["airo-robots"]}, packages=setuptools.find_packages(exclude=["test"]), entry_points={ "console_scripts": [ From 9cfc99b6eff7bb50e23bdb029e5c8fca789a8213 Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Mon, 4 Mar 2024 11:07:05 +0100 Subject: [PATCH 03/13] CI updates (#138) * install wheel package * update versions of setup & python action and install wheel package * cache pip packages * lock pytest version as fix for issue #137 --------- Co-authored-by: tlpss --- .github/workflows/pytest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index f68d6ffe..b0f6f149 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -32,7 +32,7 @@ jobs: run: | python -m pip install --upgrade pip pip install wheel setuptools - pip install pytest + pip install "pytest<8.0.0" pip install airo-typing/ airo-spatial-algebra/ airo-camera-toolkit/ airo-robots/ airo-teleop/ airo-dataset-tools/ - name: Run Tests run: pytest ${{matrix.package}}/ From 2468a96cb737e754cb47d2ee166f87e0f52f76e6 Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Mon, 4 Mar 2024 13:41:23 +0100 Subject: [PATCH 04/13] rework README (#135) * rework README * add reference to versioning docs --- README.md | 282 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 177 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index ed75a904..b3f8d3c1 100644 --- a/README.md +++ b/README.md @@ -1,75 +1,131 @@ # airo-mono -This repository contains ready-to-use python packages to accelerate the development of robotic manipulation systems. -Instead of reimplementing the same functionalities over and over, this repo provides ready-to-use implementations and aims to leverage experience by updating the implementations with best practices along the way. +Welcome to `airo-mono`! This repository provides ready-to-use Python packages to accelerate the development of robotic manipulation systems. + +**Key Motivation:** + * ๐Ÿš€ **Accelerate Experimentation:** Reduce the time spent on repetitive coding tasks, enabling faster iteration from research idea to demo on the robots. + * ๐Ÿ˜Š **Collaboration:** Promote a shared foundation of well-tested code across the lab, boosting reliability and efficiency and promoting best practices along the way. + + +Want to learn more about our vision? Check out the in-depth explanation [here](docs/about_this_repo.md) -You can read more about the scope and motivation of this repo [here](docs/about_this_repo.md). ## Overview -The repository is structured as a monorepo (hence the name) with multiple python packages. -Below is a short overview of the packages: -| Package | Description| owner | -|-------|-------|--------| -| `airo-camera-toolkit`|code for working with RGB(D) cameras, images and point clouds |@tlpss| -|`airo-dataset-tools`| code for creating, loading and working with datasets| @Victorlouisdg| -| `airo-robots`| minimal interfaces for interacting with the controllers of robot arms and grippers| @tlpss| -| `airo-spatial-algebra`|code for working with SE3 poses |@tlpss| -|`airo-teleop`| code for teleoperating robot arms |@tlpss| -| `airo-typing` |common type definitions and conventions (e.g. extrinsics matrix = camera IN world) | @tlpss | +### Packages ๐Ÿ“ฆ +The airo-mono repository employs a monorepo structure, offering multiple Python packages, each with a distinct focus: -Each package has a dedicated readme file that contains -- a more detailed overview of the functionality provided by the package -- additional installation instructions (if required) -- additional information on design decisision etc (if applicable). +| Package | Description | Owner | +| ------------------------------------------------ | --------------------------------------------------------- | -------------- | +| ๐Ÿ“ท [`airo-camera-toolkit`](airo-camera-toolkit) | RGB(D) camera, image, and point cloud processing | @tlpss | +| ๐Ÿ—๏ธ [`airo-dataset-tools`](airo-dataset-tools) | Creating, loading, and manipulating datasets | @victorlouisdg | +| ๐Ÿค– [`airo-robots`](airo-robots) | Simple interfaces for controlling robot arms and grippers | @tlpss | +| ๐Ÿ“ [`airo-spatial-algebra`](airo-spatial-algebra) | Transforms and SE3 pose conversions | @tlpss | +| ๐ŸŽฎ [`airo-teleop`](airo-teleop) | Intuitive teleoperation of robot arms | @tlpss | +| ๐Ÿ›ก๏ธ [`airo-typing`](airo-typing) | Type definitions and conventions | @tlpss | -Furthermore, each package has a 'code owner'. This is the go-to person if you: -- have questions about what is supported or about the code in general -- want to know more about why something is implemented in a particular way -- want to add new functionality to the package +**Detailed Information:** Each package contains its own dedicated README outlining: + - A comprehensive overview of the provided functionality + - Package-specific installation instructions (if needed) + - Rationale behind design choices (if applicable) -Some packages also have a command line interface. Simply run `$package-name --help` in your terminal to learn more. E.g.`$airo-dataset-tools --help`. +**Code Ownership:** The designated code owner for each package is your go-to resource for: + - Understanding features, codebase, and design decisions. ๐Ÿค” + - Proposing and contributing new package features. ๐ŸŒŸ -# Installation -There are a number of ways to install packages from this repo. As this repo is still in development and has breaking changes every now and then, we recommend locking on specific commits or releases if you need stability. -## regular install -Installs the packages from PyPI. +**Command Line Functionality:** Some packages offer command-line interfaces (CLI). +Run `package-name --help` for details. Example: `airo-dataset-tools --help` +### Sister repositories ๐ŸŒฑ +Repositories that follow the same style as `airo-mono` packages, but are not part of the monorepo (for various reasons): -not available yet -`pip install airo-camera-toolkit ....` +| Repository | Description | +| -------------------------------------------------------------- | ----------------------------------------------- | +| ๐ŸŽฅ [`airo-blender`](https://github.com/airo-ugent/airo-blender) | Synthetic data generation with Blender | +| ๐Ÿ›’ [`airo-models`](https://github.com/airo-ugent/airo-models) | Collection of robot and object models and URDFs | +| ๐Ÿ‰ `airo-drake` | Integration with Drake (coming soon) | +| ๐Ÿงญ `airo-planner` | Motion planning interfaces (coming soon) | -## installing from dev builds -Installs from dev builds, which are created for each commit on the main. +### Usage & Philosophy ๐Ÿ“– +We believe in *keep simple things simple*. Starting a new project should\* be as simple as: +```bash +pip install airo-camera-toolkit airo-robots +``` +And writing a simple script: +```python +from airo_camera_toolkit.cameras.zed.zed2i import Zed2i +from airo_robots.manipulators.hardware.ur_rtde import URrtde +from airo_robots.grippers.hardware.robotiq_2f85_urcap import Robotiq2F85 + +robot_ip_address = "10.40.0.162" + +camera = Zed2i() +robot = URrtde(ip_address=robot_ip_address) +gripper = Robotiq2F85(ip_address=robot_ip_address) + +image = camera.get_rgb_image() +grasp_pose = select_grasp_pose(image) # example: user provides grasp pose +robot.move_linear_to_tcp_pose(grasp_pose).wait() +gripper.close().wait() +``` -not availble yet. +> \* we are still simplifying the installation process and the long imports +### Projects using `airo-mono` ๐ŸŽ‰ +Probably the best way to learn what `airo-mono` has to offer, is to look at the projects it powers: +| Project | Description | +| --------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| ๐Ÿ‘• [`cloth-competition`](https://github.com/Victorlouisdg/cloth-competition) | airo-mono is the backbone of the [ICRA 2024 Cloth Competition](https://airo.ugent.be/cloth_competition/) ๐Ÿ†! | -## from github source +## Installation ๐Ÿ”ง -DEPRECATED. +### Option 1: Local clone ๐Ÿ“ฅ -We discourage the use of this installation method! +#### 1.1 Conda method +Make sure you have a version of conda e.g. [miniconda](https://docs.anaconda.com/free/miniconda/) installed. +To make the conda environment creation faster, we recommend configuring the [libmamba solver](https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community) first. -| Package | command | -|-------|-------| -|`airo-typing` |`python -m pip install 'airo-typing @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-typing'`| -|`airo-dataset-tools`|`python -m pip install 'airo-dataset-tools @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-dataset-tools'`| -|`airo-robots`|`python -m pip install 'airo-robots @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-robots'`| -|`airo-spatial-algebra`|`python -m pip install 'airo-spatial-algebra @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-spatial-algebra' `| -|`airo-teleop`|`python -m pip install 'airo-teleop @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-teleop'`| -|`airo-camera-toolkit`|`python -m pip install 'airo-camera-toolkit @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-camera-toolkit'`| +Then run the following commands: +```bash +git clone https://github.com/airo-ugent/airo-mono.git +cd airo-mono +conda env create -f environment.yaml +``` +This will create a conda environment called `airo-mono` with all packages installed. You can activate the environment with `conda activate airo-mono`. -Make sure you install the packages according to their dependency tree. If you have not installed the airo-mono packages on which a package depends first, you will get a missing import error (or it will install the package from PyPI..) +#### 1.2 Pip method +While we prefer using conda, you can also install the packages simply with pip: -## local installation -**git submodule** +```bash +git clone https://github.com/airo-ugent/airo-mono.git +cd airo-mono +pip install -e airo-robots -e airo-dataset-tools -e airo-camera-toolkit +``` +### Option 2: Installation from Github ๐ŸŒ +> โ„น๏ธ This method will be deprecated in the future, as we are moving to PyPI for package distribution. [Direct references](https://peps.python.org/pep-0440/#direct-references) are not allowed in projects that are to be published on PyPI. + +You can also install the packages from this repository directly with pip. This is mainly useful if you want to put `airo-mono` packages as dependencies in your `environment.yaml` file: +```yaml +name: my-airo-project +channels: + - conda-forge +dependencies: + - python=3.10 + - pip + - pip: + - "airo-typing @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-typing" + - "airo-spatial-algebra @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-spatial-algebra" + - "airo-camera-toolkit @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-camera-toolkit" + - "airo-dataset-tools @ git+https://github.com/airo-ugent/airo-mono@main#subdirectory=airo-dataset-tools" +``` + +### Option 3: Git submodule ๐Ÿš‡ You can add this repo as a submodule and install the relevant packages afterwards with regular pip commands. This allows to seamlessly make contributions to this repo whilst working on your own project or if you want to pin on a specific version. In your repo, run: -``` +```bash git submodule init git submodule add https://github.com/airo-ugent/airo-mono@ cd airo-mono @@ -77,89 +133,105 @@ cd airo-mono You can now add the packages you need to your requirements or environment file, either in development mode or through a regular pip install. More about submodules can be found [here](https://git-scm.com/book/en/v2/Git-Tools-Submodules). Make sure to install the packages in one pip command such that pip can install them in the appropriate order to deal with internal dependencies. +### Option 4: Installation from PyPI ๐Ÿ“ฆ +> ๐Ÿšง Not available yet, but coming soon. -# Developer guide -### setting up local environment -To set up your development environment after cloning this repo, run: +Install the packages from PyPI. +``` +pip install airo-camera-toolkit airo-dataset-tools airo-robots +``` + +## Developer guide ๐Ÿ› ๏ธ +### Setup +Create and activate the conda environment, then run: ``` -conda env create -f environment.yaml -conda activate airo-mono pip install -r dev-requirements.txt pre-commit install ``` -### Coding style -Formatting is done with black (code style), isort (sort imports) and autoflake (remove unused imports and variables). Flake8 is used as linter. These are bundled with [pre-commit](https://pre-commit.com/) as configured in the `.pre-commit-config.yaml` file. You can manually run pre-commit with `pre-commit run -a`. +### Coding style ๐Ÿ‘ฎ +We use [pre-commit](https://pre-commit.com/) to automatically enforce coding standards before every commit. -Packages can be typed (optional, but strongly recommended). For this, mypy is used. To run mypy on a package: `mypy `. +Our [.pre-commit-config.yaml](.pre-commit-config.yaml) file defines the tools and checks we want to run: + - **Formatting**: Black, isort, and autoflake + - **Linting**: Flake8 -Docstrings should be formatted in the [google docstring format](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). +**Typing:** Packages can be typed (optional, but strongly recommended). For this, mypy is used. Note that pre-commit curretnly does not run mypy, so you should run it manually with `mypy `, e.g. `mypy airo-camera-toolkit`. -### Testing -Testing is done with pytest (as this is more flexible than unittest). Tests should be grouped per package, as the CI pipeline will run them for each package in isolation. Also note that there should always be at least one test, since pytest will otherwise [throw an error](https://github.com/pytest-dev/pytest/issues/2393). +**Docstrings:** Should be in the [google docstring format](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings). -You can manually run the tests for a package with `make pytest `. This will also provide a coverage report. +### Testing ๐Ÿงช + - **Framework:** Pytest for its flexibility. + - **Organization:** Tests are grouped by package, the CI pipeline runs them in isolation + - **Minimum:** Always include [at least one test per package](https://github.com/pytest-dev/pytest/issues/2393). + - **Running Tests:** `make pytest ` (includes coverage report). + - **Hardware Testing:** Cameras and robots have scripts available for manual testing. These scripts provide simple sanity checks to verify connections and functionality. -Testing hardware interfaces etc with unittests is rather hard, but we expect all other code to have tests. Testing not only formalises the ad-hoc playing that you usually do whilst developing, it also enables anyone to later on refactor the code and quickly check if this did not break anything. -For hardware-related code, we expect 'user tests' to be avaiable. By this we mean a script that clearly states what should happen with the hardware, so that someone can connect to the hardware and quickly see if everything is working as expected. -### CI -we use github actions to do the following checks on each PR, push to master (and push to a branch called CI for doing development on the CI pipeline itself) +### Continuous Integration (CI) โš™๏ธ +We use GitHub Actions to run the following checks: -- formatting check -- mypy static type checking -- pytest unittests. +| Workflow | Runs When | +| ----------------------------------------------- | ---------------------------------------------- | +| [pre-commit](.github/workflows/pre-commit.yaml) | Every push | +| [mypy](.github/workflows/mypy.yaml) | pushes to `main`, PRs and the `ci-dev` branch | +| [pytest](.github/workflows/pytest.yaml) | pushes to `main`, PRs and the `ci-dev` branch | -The tests are executed for each package in isolation using [github actions Matrices](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs), which means that only that package and its dependencies are installed in an environment to make sure each package correctly declares its dependencies. The downside is that this has some overhead in creating the environments, so we should probably look into caching them once the runtime becomes longer. -### Management of (local) dependencies +**Package Test Isolation:** We use [Github Actions matrices](https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs) to run tests for each package in its own environment. This ensures packages correctly declare their dependencies. However, creating these environments adds overhead, so we'll explore caching strategies to optimize runtime as complexity increases. -An issue with using a monorepo is that you want to have packages declare their local dependencies as well. But before you publish your packages or if you want to test unreleased code (as usually), this creates an issue: where should pip find these local package? Though there exist more advanced package managers such as Poetry, ([more background on package and dependency management in python](https://ealizadeh.com/blog/guide-to-python-env-pkg-dependency-using-conda-poetry/) -) that can handle this, we have opted to stick with pip to keep the barier for new developers lower. +### Creating a new package โœจ -This implies we simply add local dependencies in the setup file as regular dependencies, but we have to make sure pip can find the dependencies when installing the pacakges.There are two options to do so: -1. You make sure that the local dependencies are installed before installing the package, either by running the pip install commands along the dependency tree, or by running all installs in a single pip commamd: `pip install ` -2. you create distributions for the packages upfront and then tell pip where to find them (because they won't be on PyPI, which is where pip searches by default): `pip install --find-link https:// or /path/to/distributions/dir` +To quickly setup up Python projects you can use this [cookiecutter template](https://github.com/tlpss/research-template). In the future we might create a similar one for `airo-mono` packages. For now here are the steps you have to take: - -Initially, we used a direct link to point to the path of the dependencies, but this created some issues and hence we now use this easier approach. see [#91](https://github.com/airo-ugent/airo-mono/issues/91) for more details. - -### Creating a new package -Creating a new package is kind of a hassle atm, in the future we might add a coockiecutter template for it. For now here are the steps you have to take: -- create the nested structure +1. **Create directory structure:** ``` -/ - / - code.py - test/ - testx.py - README.md - setup.py +airo-package/ + โ”œโ”€ airo_package/ + โ”‚ โ””โ”€ code.py + โ”œโ”€ test/ + โ”‚ โ””โ”€ test_some_feature.py + โ”œโ”€ README.md + โ””โ”€ setup.py ``` -- create the minimal setup.py - - handle internal dependencies with extra_requires {'external'} -- add package name to matrix of CI flows -- add package to top-level readme [here](#overview) -- add package to the `environment.yaml` -- add package to the [install script](scripts/install-airo-mono.sh) +2. **Integrate with CI:** update the CI workflow matrices to include your package. +3. **Update Documentation:** add your package to this README +4. **Installation:** add package to the `environment.yaml` +and `scripts/install-airo-mono.sh`. + +### Command Line Interfaces ๐Ÿ’ป +For convenient access to specific functions, we provide command-line interfaces (CLIs). This lets you quickly perform tasks like visualizing COCO datasets or starting hand-eye calibration without the need to write Python scripts to change arguments (e.g., changing a data path or robot IP address). + + + - **Framework:** [Click](https://click.palletsprojects.com/en/8.1.x/) for composable CLIs. + - **Exposure:** We use Setuptools [`console_scripts`](https://setuptools.pypa.io/en/latest/userguide/entry_point.html) to make commands available. + - **Organization:** User-facing CLI commands belong in a top-level `cli.py` file. + - **Separation:** Keep CLI code distinct from core functionality for maximum flexibility. + - **Example:** [`airo_dataset_tools/cli.py`](airo-dataset-tools/airo_dataset_tools/cli.py) and [`airo-dataset-tools/setup.py`](airo-dataset-tools/setup.py). + - **Developer Focus:** Scripts' `__main__()` functions can still house developer-centric CLIs. Consider moving user-friendly ones to the package CLI. -### Command Line Interfaces -It can become convenient to expose certain functionality as a command line interface (CLI). -E.g. if you have a fucntion that visualizes a coco dataset, you might want to make it easy for the user to use this function on an arbitrary dataset (path), without requiring the user to create a python script that calls that function with the desired arguments. +### Versioning & Releasing ๐Ÿท๏ธ -We use [click](https://click.palletsprojects.com/en/8.1.x/) to create CLIs and make use of the setuptools `console_scripts` to conveniently expose them to the user, which allows to do `$ package-name command --options/arguments` in your terminal instead of having to manually point to the location of the python file when invoking the command. +As a first step towards PyPI releases of the `airo-mono` packages, we have already started versioning them. +Read more about it in [docs/versioning.md](docs/versioning.md). -All CLI commands of a package that are meant to be used by end-users should be grouped in a top-level `cli.py` file. It is preferred to separate the CLI command implmmentation from the actual implementation of the functionality, so that the funcionality can still be used by other python code. +### Design choices โœ๏ธ +- **Minimalism:** Before coding, explore existing libraries. Less code means easier maintenance. +- **Properties:** Employ Python properties ([@property](https://docs.python.org/3/howto/descriptor.html#properties)) for getters/setters. This enhances user interaction and unlocks powerful code patterns. +- **Logging**: Use [loguru](https://loguru.readthedocs.io/en/stable/) for structured debugging instead of print statements. +- **Output Data:** Favor native datatypes or NumPy arrays for easy compatibility. For more complex data, use dataclasses as in [airo-typing](airo-typing). -For an example, you should take a look at the `airo_dataset_tools/cli.py` file and the `airo-dataset-tools/setup.py`. Make sure to read through the docs of the click package to understand what is happening. +#### Management of local dependencies in a Monorepo +> **TODO:** simplify this explanation and move it to the setup or installation section. + +An issue with using a monorepo is that you want to have packages declare their local dependencies as well. But before you publish your packages or if you want to test unreleased code (as usually), this creates an issue: where should pip find these local package? Though there exist more advanced package managers such as Poetry, ([more background on package and dependency management in python](https://ealizadeh.com/blog/guide-to-python-env-pkg-dependency-using-conda-poetry/) +) that can handle this, we have opted to stick with pip to keep the barier for new developers lower. -Also note that you can still have a `__main__` block in a python module with a CLI command, but those should be more for developers than for end users. If you expect it to be useful for end-users, you might want to consider moving it to the package CLI. -### Design choices -- class attributes that require getter/setter should use python [properties](https://realpython.com/python-property/). This is not only more pythonic (for end-user engagement with the attribute), but more importantly it enables a whole bunch of better code patterns as you can override a decorator, but not a plain attribute. -- the easiest code to maintain is no code at all. So thorougly consider if the functionality you want does not already have a good implementation and could be imported with a reasonable dependency cost. -- it is strongly encouraged to use [click](https://click.palletsprojects.com/en/8.1.x/) for all command line interfaces. -- it is strongly advised to use logging instead of print statements. [loguru](https://loguru.readthedocs.io/en/stable/) is the recommended logging tool. -- Use python dataclasses for configuration, instead of having a ton of argument in the build/init functions of your classes. -- The output of operations should preferably be in a `common` format. E.g. if you have a method that creates a pointcloud, we prefer that method to return a numpy array instead of your own pointcloud class, that internally contains such an array. You can of course use such classes internally, but we prefer that the end user gets 'common formats', because he/she is most likely to immediately extract the numpy arrar out of your custom data class anyways. +This implies we simply add local dependencies in the setup file as regular dependencies, but we have to make sure pip can find the dependencies when installing the pacakges.There are two options to do so: +1. You make sure that the local dependencies are installed before installing the package, either by running the pip install commands along the dependency tree, or by running all installs in a single pip commamd: `pip install ` +2. you create distributions for the packages upfront and then tell pip where to find them (because they won't be on PyPI, which is where pip searches by default): `pip install --find-link https:// or /path/to/distributions/dir` + + +Initially, we used a direct link to point to the path of the dependencies, but this created some issues and hence we now use this easier approach. see [#91](https://github.com/airo-ugent/airo-mono/issues/91) for more details. From 30f28d8c70c9ab3dbdff80407c463c8827b83709 Mon Sep 17 00:00:00 2001 From: Victorlouisdg Date: Mon, 4 Mar 2024 14:14:45 +0100 Subject: [PATCH 05/13] Link to airo-drake in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b3f8d3c1..e0bc3c22 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ Repositories that follow the same style as `airo-mono` packages, but are not par | -------------------------------------------------------------- | ----------------------------------------------- | | ๐ŸŽฅ [`airo-blender`](https://github.com/airo-ugent/airo-blender) | Synthetic data generation with Blender | | ๐Ÿ›’ [`airo-models`](https://github.com/airo-ugent/airo-models) | Collection of robot and object models and URDFs | -| ๐Ÿ‰ `airo-drake` | Integration with Drake (coming soon) | +| ๐Ÿ‰ [`airo-drake`](https://github.com/airo-ugent/airo-drake) | Integration with Drake | | ๐Ÿงญ `airo-planner` | Motion planning interfaces (coming soon) | ### Usage & Philosophy ๐Ÿ“– From 087da4155e908846cadbc145df3d17f11ceb885d Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Mon, 4 Mar 2024 17:36:14 +0100 Subject: [PATCH 06/13] Add entity path to MultiprocessRGB[D]RerunLogger (#136) * Add entity path to MultiprocessRGB[D]RerunLogger * Add entity path for the depth image to MultiprocessRGBDRerunLogger * Update CHANGELOG.md --- CHANGELOG.md | 2 +- .../multiprocess/multiprocess_rerun_logger.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bbb9f312..24a9bf81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly - `BoundingBox3DType` - `Zed2i.ULTRA_DEPTH_MODE` to enable the ultra depth setting for the Zed2i cameras - `OpenCVVideoCapture` implementation of `RGBCamera` for working with arbitrary cameras - +- `MultiprocessRGBRerunLogger` and `MultiprocessRGBDRerunLogger` now allow you to pass an `entity_path` value which determines where the RGB and depth images will be logged ### Changed - Dropped support for python 3.8 and added 3.11 to the testing matrix [#103](https://github.com/airo-ugent/airo-mono/issues/103). diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rerun_logger.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rerun_logger.py index b34223eb..1f627ee2 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rerun_logger.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rerun_logger.py @@ -18,6 +18,7 @@ def __init__( shared_memory_namespace: str, rerun_application_id: str = "rerun", image_transform: Optional[ImageTransform] = None, + entity_path: Optional[str] = None, ): super().__init__(daemon=True) self._shared_memory_namespace = shared_memory_namespace @@ -25,6 +26,9 @@ def __init__( self._rerun_application_id = rerun_application_id self._image_transform = image_transform + # If the entity path is not given, we use the `_shared_memory_namespace` value as entity path (maintaining backwards compatibility). + self._entity_path = entity_path if entity_path is not None else shared_memory_namespace + def _log_rgb_image(self) -> None: import rerun as rr @@ -39,7 +43,7 @@ def _log_rgb_image(self) -> None: if self._image_transform is not None: image_rgb = self._image_transform.transform_image(image_rgb) - rr.log(self._shared_memory_namespace, rr.Image(image_rgb).compress(jpeg_quality=90)) + rr.log(self._entity_path, rr.Image(image_rgb).compress(jpeg_quality=90)) def run(self) -> None: """main loop of the process, runs until the process is terminated""" @@ -63,13 +67,18 @@ def __init__( shared_memory_namespace: str, rerun_application_id: str = "rerun", image_transform: Optional[ImageTransform] = None, + entity_path: Optional[str] = None, + entity_path_depth: Optional[str] = None, ): super().__init__( shared_memory_namespace, rerun_application_id, image_transform, + entity_path, ) + self._entity_path_depth = entity_path_depth if entity_path_depth is not None else f"{self._entity_path}_depth" + def _log_depth_image(self) -> None: import rerun as rr @@ -78,7 +87,7 @@ def _log_depth_image(self) -> None: depth_image = self._receiver.get_depth_image() if self._image_transform is not None: depth_image = self._image_transform.transform_image(depth_image) - rr.log(f"{self._shared_memory_namespace}_depth", rr.Image(depth_image).compress(jpeg_quality=90)) + rr.log(self._entity_path_depth, rr.Image(depth_image).compress(jpeg_quality=90)) def run(self) -> None: """main loop of the process, runs until the process is terminated""" From e16ab7f997283e850aa5765d96477f1c4e1914fa Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Thu, 7 Mar 2024 16:13:44 +0100 Subject: [PATCH 07/13] Point cloud transform (#141) * Add transform_point_cloud * Update CHANGELOG.md --- CHANGELOG.md | 2 +- .../point_clouds/operations.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24a9bf81..185e3e90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly ### Added - `PointCloud` dataclass as the main data structure for point clouds in airo-mono - Notebooks to get started with point clouds, checking performance and logging to rerun -- Functions to crop point clouds and filter points with a mask (e.g. low-confidence points) +- Functions to crop point clouds, filter points with a mask (e.g. low-confidence points), and transform point clouds - Functions to convert from our numpy-based dataclass to and from open3d point clouds - `BoundingBox3DType` - `Zed2i.ULTRA_DEPTH_MODE` to enable the ultra depth setting for the Zed2i cameras diff --git a/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py b/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py index d9c0891e..17ab1157 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py +++ b/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py @@ -1,7 +1,8 @@ from typing import Any import numpy as np -from airo_typing import BoundingBox3DType, PointCloud +from airo_spatial_algebra.operations import transform_points +from airo_typing import BoundingBox3DType, HomogeneousMatrixType, PointCloud def filter_point_cloud(point_cloud: PointCloud, mask: Any) -> PointCloud: @@ -61,3 +62,17 @@ def crop_point_cloud( """ crop_mask = generate_point_cloud_crop_mask(point_cloud, bounding_box) return filter_point_cloud(point_cloud, crop_mask.nonzero()) + + +def transform_point_cloud(point_cloud: PointCloud, frame_transformation: HomogeneousMatrixType) -> PointCloud: + """Creates a new point cloud for which the points are transformed to the desired frame. + Will keep colors and attributes if they are present. + + Args: + point_cloud: The point cloud to transform. + frame_transformation: The transformation matrix from the current point cloud frame to the new desired frame. + + Returns: + The new transformed point cloud.""" + new_points = transform_points(frame_transformation, point_cloud.points) + return PointCloud(new_points, point_cloud.colors, point_cloud.attributes) From 1380033b17527829778324d2ca182f001c0a1bd1 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Fri, 8 Mar 2024 10:15:01 +0100 Subject: [PATCH 08/13] Add an example to the transform_point_cloud docstring --- .../airo_camera_toolkit/point_clouds/operations.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py b/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py index 17ab1157..981b4f8c 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py +++ b/airo-camera-toolkit/airo_camera_toolkit/point_clouds/operations.py @@ -68,6 +68,12 @@ def transform_point_cloud(point_cloud: PointCloud, frame_transformation: Homogen """Creates a new point cloud for which the points are transformed to the desired frame. Will keep colors and attributes if they are present. + The `frame_transformation` is a homogeneous matrix expressing the current point cloud frame in the target point cloud frame. + For example, if you capture a point cloud from a camera with the extrinsics matrix `X_W_C`, expressing the camera's pose in + the world frame, then you can express the point cloud in the world frame with: + + `point_cloud_in_world = transform_point_cloud(point_cloud, X_W_C)` + Args: point_cloud: The point cloud to transform. frame_transformation: The transformation matrix from the current point cloud frame to the new desired frame. From c81daacea7897806adacfe254e866adb01adf8b7 Mon Sep 17 00:00:00 2001 From: Victor-Louis De Gusseme Date: Mon, 25 Mar 2024 11:38:13 +0100 Subject: [PATCH 09/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0bc3c22..a884d4b0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Repositories that follow the same style as `airo-mono` packages, but are not par | ๐ŸŽฅ [`airo-blender`](https://github.com/airo-ugent/airo-blender) | Synthetic data generation with Blender | | ๐Ÿ›’ [`airo-models`](https://github.com/airo-ugent/airo-models) | Collection of robot and object models and URDFs | | ๐Ÿ‰ [`airo-drake`](https://github.com/airo-ugent/airo-drake) | Integration with Drake | -| ๐Ÿงญ `airo-planner` | Motion planning interfaces (coming soon) | +| ๐Ÿงญ [`airo-planner`](https://github.com/airo-ugent/airo-planner) | Motion planning interfaces | ### Usage & Philosophy ๐Ÿ“– We believe in *keep simple things simple*. Starting a new project should\* be as simple as: From dc8b7a77df813c7fb1a6df127c12923749339d3a Mon Sep 17 00:00:00 2001 From: tlpss Date: Thu, 13 Jun 2024 18:03:46 +0200 Subject: [PATCH 10/13] Add single polygon method to binarymask, required for YOLO as it accepts a single polygon as mask --- CHANGELOG.md | 2 + .../coco_tools/coco_instances_to_yolo.py | 8 +- .../coco_tools/transform_dataset.py | 19 +++- .../segmentation_mask_converter.py | 102 +++++++++++++++++- 4 files changed, 117 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 185e3e90..6df2902b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly ### Added +- add method `as_single_polygon` to combine disconnected parts of a binary mask into a single polygon to the `Mask` class, useful for data formats that only allow for a single polygon such as YOLO. - `PointCloud` dataclass as the main data structure for point clouds in airo-mono - Notebooks to get started with point clouds, checking performance and logging to rerun - Functions to crop point clouds, filter points with a mask (e.g. low-confidence points), and transform point clouds @@ -25,6 +26,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly - `MultiprocessRGBRerunLogger` and `MultiprocessRGBDRerunLogger` now allow you to pass an `entity_path` value which determines where the RGB and depth images will be logged ### Changed +- `coco-to-yolo` conversion now creates a single polygon of all disconnected parts of the mask instead of simply taking the first polygon of the list. - Dropped support for python 3.8 and added 3.11 to the testing matrix [#103](https://github.com/airo-ugent/airo-mono/issues/103). - Set python version to 3.10 because of an issue with the `ur_rtde` wheels [#121](https://github.com/airo-ugent/airo-mono/issues/121). Updated README.md to reflect this change. diff --git a/airo-dataset-tools/airo_dataset_tools/coco_tools/coco_instances_to_yolo.py b/airo-dataset-tools/airo_dataset_tools/coco_tools/coco_instances_to_yolo.py index e8714210..949fccee 100644 --- a/airo-dataset-tools/airo_dataset_tools/coco_tools/coco_instances_to_yolo.py +++ b/airo-dataset-tools/airo_dataset_tools/coco_tools/coco_instances_to_yolo.py @@ -92,18 +92,16 @@ def create_yolo_dataset_from_coco_instances_dataset( yolo_id = yolo_category_index.index(category) if use_segmentation: segmentation = annotation.segmentation - # convert to polygon if required + # convert to **single** polygon segmentation = BinarySegmentationMask.from_coco_segmentation_mask(segmentation, width, height) - segmentation = segmentation.as_polygon + segmentation = segmentation.as_single_polygon if segmentation is None: # should actually never happen as each annotation is assumed to have a segmentation if you pass use_segmentation=True # but we filter it for convenience to deal with edge cases print(f"skipping annotation for image {image_path}, as it has no segmentation") continue - segmentation = segmentation[ - 0 - ] # only use first polygon, since coco does not support multiple polygons? + file.write(f"{yolo_id}") for (x, y) in zip(segmentation[0::2], segmentation[1::2]): file.write(f" {x/width} {y/height}") diff --git a/airo-dataset-tools/airo_dataset_tools/coco_tools/transform_dataset.py b/airo-dataset-tools/airo_dataset_tools/coco_tools/transform_dataset.py index 302be4d4..8bad3244 100644 --- a/airo-dataset-tools/airo_dataset_tools/coco_tools/transform_dataset.py +++ b/airo-dataset-tools/airo_dataset_tools/coco_tools/transform_dataset.py @@ -3,9 +3,9 @@ from typing import Any, Callable, List, Optional import albumentations as A +import cv2 import numpy as np import tqdm -from airo_dataset_tools.coco_tools.transforms import PillowResize from airo_dataset_tools.data_parsers.coco import ( CocoImage, CocoInstanceAnnotation, @@ -90,7 +90,15 @@ def apply_transform_to_coco_dataset( # type: ignore # noqa: C901 # convert coco keypoints to list of (x,y) keypoints if transform_bbox: - all_bboxes.append(annotation.bbox) + bbox = annotation.bbox + # set bbox width, height to at least 1 + if bbox[3] < 1 or bbox[2] < 1: + # x_min must be < x_max for albumentations check + bbox = [0, 0, 1, 1] + print( + f"Invalid bbox for image {coco_image.file_name} and annotation {annotation.id}. Setting to [0,0,1,1]" + ) + all_bboxes.append(bbox) if transform_segmentation: # convert segmentation to binary mask @@ -123,7 +131,10 @@ def apply_transform_to_coco_dataset( # type: ignore # noqa: C901 if not os.path.exists(transformed_image_dir): os.makedirs(transformed_image_dir) # specify quality to use for JPEG, (format is determined by file extension) - transformed_image.save(os.path.join(target_image_path, coco_image.file_name), quality=95) + # transformed_image.save(os.path.join(target_image_path, coco_image.file_name)) + # convert to BGR for opencv + transformed_image_cv2 = cv2.cvtColor(np.array(transformed_image), cv2.COLOR_RGB2BGR) + cv2.imwrite(os.path.join(target_image_path, coco_image.file_name), transformed_image_cv2) # change the metadata of the image coco object coco_image.width = transformed_image.width @@ -184,7 +195,7 @@ def resize_coco_dataset( transformed_dataset_dir = target_dataset_dir os.makedirs(transformed_dataset_dir, exist_ok=True) - transforms = [PillowResize(height, width)] + transforms = [A.Resize(height, width)] coco_json = json.load(open(annotations_json_path, "r")) coco_dataset = CocoInstancesDataset(**coco_json) transformed_dataset = apply_transform_to_coco_dataset( diff --git a/airo-dataset-tools/airo_dataset_tools/segmentation_mask_converter.py b/airo-dataset-tools/airo_dataset_tools/segmentation_mask_converter.py index 6105b563..ec44244f 100644 --- a/airo-dataset-tools/airo_dataset_tools/segmentation_mask_converter.py +++ b/airo-dataset-tools/airo_dataset_tools/segmentation_mask_converter.py @@ -8,6 +8,80 @@ from pycocotools import mask +def merge_multi_segment(segments): + """ + + code taken from: https://github.com/ultralytics/JSON2YOLO/blob/main/general_json2yolo.py#L330 + (AGPL licensed.) + + Merge multi segments to one list. Find the coordinates with min distance between each segment, then connect these + coordinates with one thin line to merge all segments into one. + + This is useful to convert masks to yolo dataset, as yolo only supports one polygon for each object. + + Args: + segments(List(List)): original segmentations in coco's json file. + like [segmentation1, segmentation2,...], + each segmentation is a list of coordinates. + + Returns: + List(List): merged segments. + """ + + def min_index(arr1, arr2): + """ + Find a pair of indexes with the shortest distance. + + Args: + arr1: (N, 2). + arr2: (M, 2). + Return: + a pair of indexes(tuple). + """ + dis = ((arr1[:, None, :] - arr2[None, :, :]) ** 2).sum(-1) + return np.unravel_index(np.argmin(dis, axis=None), dis.shape) + + s = [] + segments = [np.array(i).reshape(-1, 2) for i in segments] + idx_list = [[] for _ in range(len(segments))] + + # record the indexes with min distance between each segment + for i in range(1, len(segments)): + idx1, idx2 = min_index(segments[i - 1], segments[i]) + idx_list[i - 1].append(idx1) + idx_list[i].append(idx2) + + # use two round to connect all the segments + for k in range(2): + # forward connection + if k == 0: + for i, idx in enumerate(idx_list): + # middle segments have two indexes + # reverse the index of middle segments + if len(idx) == 2 and idx[0] > idx[1]: + idx = idx[::-1] + segments[i] = segments[i][::-1, :] + + segments[i] = np.roll(segments[i], -idx[0], axis=0) + segments[i] = np.concatenate([segments[i], segments[i][:1]]) + # deal with the first segment and the last one + if i in [0, len(idx_list) - 1]: + s.append(segments[i]) + else: + idx = [0, idx[1] - idx[0]] + s.append(segments[i][idx[0] : idx[1] + 1]) + + else: + for i in range(len(idx_list) - 1, -1, -1): + if i not in [0, len(idx_list) - 1]: + idx = idx_list[i] + nidx = abs(idx[1] - idx[0]) + s.append(segments[i][nidx:]) + + s = np.concatenate(s, axis=0).reshape(-1).tolist() + return s + + class BinarySegmentationMask: """Class that holds a binay segmentation mask and can convert between binary bitmask and/or the different COCO segmentation formats: - polygon: [list[list[float]]] containing [x,y] coordinates of the polygon(s) @@ -72,6 +146,18 @@ def as_polygon(self) -> Optional[List[Polygon]]: return None return segmentation + @property + def as_single_polygon(self) -> Optional[List[Polygon]]: + """Convert a bitmap to a single polygon. If the bitmap contains multiple segments, they will be merged into one polygon.""" + poly = self.as_polygon + if poly is None: + return None + + if len(poly) == 1: + return poly[0] + poly = merge_multi_segment(poly) + return poly + @property def as_uncompressed_rle(self) -> RLEDict: raise NotImplementedError @@ -86,8 +172,14 @@ def as_compressed_rle(self) -> RLEDict: return encoded_rle -# if __name__ == "__main__": -# mask = np.zeros((10, 10)).astype(np.uint8) -# mask[1:3] = 1 -# print(mask) -# print(BinarySegmentationMask(mask).as_polygon) +if __name__ == "__main__": + m = np.zeros((10, 10)).astype(np.uint8) + m[1:3] = 1 + m[6:8, 1:5] = 1 + print(m) + print(BinarySegmentationMask(m).as_polygon) + poly = BinarySegmentationMask(m).as_single_polygon + # poly = merge_multi_segment(poly) + print(poly) + mask2 = BinarySegmentationMask.from_coco_segmentation_mask(poly, 10, 10) + print(mask2.bitmap) From e011de682ae6e047372ca0d64f7421982424f168 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 26 Jun 2024 10:14:18 +0200 Subject: [PATCH 11/13] Take ownership of airo-camera-toolkit As per our meeting on March 25th --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a884d4b0..cb6a9140 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The airo-mono repository employs a monorepo structure, offering multiple Python | Package | Description | Owner | | ------------------------------------------------ | --------------------------------------------------------- | -------------- | -| ๐Ÿ“ท [`airo-camera-toolkit`](airo-camera-toolkit) | RGB(D) camera, image, and point cloud processing | @tlpss | +| ๐Ÿ“ท [`airo-camera-toolkit`](airo-camera-toolkit) | RGB(D) camera, image, and point cloud processing | @m-decoster | | ๐Ÿ—๏ธ [`airo-dataset-tools`](airo-dataset-tools) | Creating, loading, and manipulating datasets | @victorlouisdg | | ๐Ÿค– [`airo-robots`](airo-robots) | Simple interfaces for controlling robot arms and grippers | @tlpss | | ๐Ÿ“ [`airo-spatial-algebra`](airo-spatial-algebra) | Transforms and SE3 pose conversions | @tlpss | From 2204556e9313774f168882b9ff68c4f16224af4d Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 26 Jun 2024 10:18:55 +0200 Subject: [PATCH 12/13] Shared memory improvements (#144) * Reuse shared memory file if it still exists * Unlink SHM and re-create if file exists * Reduce sleep for shm creation to by factor 10 * Logger statement instead of print * Update changelog * Uncomment accidentally commented block of code --- CHANGELOG.md | 2 ++ .../multiprocess/multiprocess_rgb_camera.py | 18 ++++++++++++++---- .../multiprocess/multiprocess_rgbd_camera.py | 8 +++++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6df2902b..52fa18c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,8 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly - Added `__init__.py` to `realsense` and `utils` in `airo_camera_toolkit.cameras`, fixing installs with pip and issue #113. - Fixed bug that returned a transposed resolution in `MultiprocessRGBReceiver`. - Using `Zed2i.PERFORMANCE_DEPTH_MODE` will now correctly use the performance mode instead of the quality mode. +- Shared memory files that were not properly cleaned up are now unlinked and then recreated. +- The wait interval for shared memory files has been reduced to .5 seconds (from 5), to speed up application start times. ### Removed - `ColoredPointCloudType` diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgb_camera.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgb_camera.py index 12ca45de..c0e2c208 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgb_camera.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgb_camera.py @@ -9,6 +9,7 @@ from airo_camera_toolkit.interfaces import RGBCamera from airo_camera_toolkit.utils.image_converter import ImageConverter from airo_typing import CameraIntrinsicsMatrixType, CameraResolutionType, NumpyFloatImageType, NumpyIntImageType +from loguru import logger _RGB_SHM_NAME = "rgb" _RGB_SHAPE_SHM_NAME = "rgb_shape" @@ -28,7 +29,16 @@ def shared_memory_block_like(array: np.ndarray, name: str) -> Tuple[shared_memor Returns: The created shared memory block and a new array that is backed by the shared memory block. """ - shm = shared_memory.SharedMemory(create=True, size=array.nbytes, name=name) + try: + shm = shared_memory.SharedMemory(create=True, size=array.nbytes, name=name) + except FileExistsError: + logger.warning(f"Shared memory file {name} exists. Will unlink and re-create it.") + + shm_old = shared_memory.SharedMemory(create=False, size=array.nbytes, name=name) + shm_old.unlink() + + shm = shared_memory.SharedMemory(create=True, size=array.nbytes, name=name) + shm_array: np.ndarray = np.ndarray(array.shape, dtype=array.dtype, buffer=shm.buf) shm_array[:] = array[:] return shm, shm_array @@ -203,10 +213,10 @@ def __init__( is_shm_found = True break except FileNotFoundError: - print( - f'INFO: SharedMemory namespace "{self._shared_memory_namespace}" not found yet, retrying in 5 seconds.' + logger.debug( + f'SharedMemory namespace "{self._shared_memory_namespace}" not found yet, retrying in .5 seconds.' ) - time.sleep(5) + time.sleep(0.5) if not is_shm_found: raise FileNotFoundError("Shared memory not found.") diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgbd_camera.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgbd_camera.py index bc38acb1..d1f6b252 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgbd_camera.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgbd_camera.py @@ -121,10 +121,10 @@ def __init__(self, shared_memory_namespace: str) -> None: is_shm_found = True break except FileNotFoundError: - print( - f'INFO: SharedMemory namespace "{self._shared_memory_namespace}" (RGBD) not found yet, retrying in 5 seconds.' + logger.debug( + f'SharedMemory namespace "{self._shared_memory_namespace}" not found yet, retrying in .5 seconds.' ) - time.sleep(5) + time.sleep(0.5) if not is_shm_found: raise FileNotFoundError("Shared memory not found.") @@ -159,6 +159,8 @@ def _retrieve_depth_image(self) -> NumpyIntImageType: def _close_shared_memory(self) -> None: """Closing shared memory signal that""" + super()._close_shared_memory() + self.depth_shm.close() self.depth_image_shm.close() From f78efa3af7e86b48a906e920924a8d0c88e74037 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Mon, 8 Jul 2024 11:54:10 +0200 Subject: [PATCH 13/13] CHANGELOD.md updated to fix Github Actions error --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52fa18c0..ca5bdf10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly - `MultiprocessRGBRerunLogger` and `MultiprocessRGBDRerunLogger` now allow you to pass an `entity_path` value which determines where the RGB and depth images will be logged ### Changed -- `coco-to-yolo` conversion now creates a single polygon of all disconnected parts of the mask instead of simply taking the first polygon of the list. +- `coco-to-yolo` conversion now creates a single polygon of all disconnected parts of the mask instead of simply taking the first polygon of the list. - Dropped support for python 3.8 and added 3.11 to the testing matrix [#103](https://github.com/airo-ugent/airo-mono/issues/103). - Set python version to 3.10 because of an issue with the `ur_rtde` wheels [#121](https://github.com/airo-ugent/airo-mono/issues/121). Updated README.md to reflect this change.