From efd3e20063f77b581da4ac57d17475eb536f30c6 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 08:27:25 +0100 Subject: [PATCH 1/9] OpenCVVideoCapture implementation based on #81 --- .../airo_camera_toolkit/cameras/README.md | 2 + .../cameras/generic_opencv/__init__.py | 0 .../cameras/generic_opencv/generic_opencv.py | 80 +++++++++++++++++++ .../generic_opencv/generic_opencv_camera.md | 6 ++ 4 files changed, 88 insertions(+) create mode 100644 airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/__init__.py create mode 100644 airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py create mode 100644 airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv_camera.md diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md b/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md index db373564..8867e288 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md @@ -7,6 +7,8 @@ This subpackage contains implementations of the camera interface for the cameras It also contains code to enable multiprocessed use of the camera streams: [multiprocessed camera](./multiprocess/) +There is also an implementation for generic RGB cameras using OpenCV `VideoCapture`: [generic OpenCV camera](./generic_opencv/) + ## 1. Installation Implementations usually require the installation of SDKs, drivers etc. to communicate with the camera. This information can be found in `READMEs` for each camera: diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/__init__.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py new file mode 100644 index 00000000..d21ebd26 --- /dev/null +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import math +from typing import Any, Optional, Tuple + +import cv2 +from airo_camera_toolkit.interfaces import RGBCamera +from airo_camera_toolkit.utils.image_converter import ImageConverter +from airo_typing import CameraIntrinsicsMatrixType, CameraResolutionType, NumpyFloatImageType, NumpyIntImageType + + +class OpenCVVideoCapture(RGBCamera): + """Wrapper around OpenCV's VideoCapure so we can test the camera interface without external cameras.""" + + def __init__( + self, video_capture_args: Tuple[Any] = (0,), intrinsics_matrix: Optional[CameraIntrinsicsMatrixType] = None + ) -> None: + self.video_capture = cv2.VideoCapture(*video_capture_args) + if not self.video_capture.isOpened(): + raise RuntimeError("Cannot open camera") + + self.fps = self.video_capture.get(cv2.CAP_PROP_FPS) + self._intrinsics_matrix = intrinsics_matrix + + self._resolution = ( + math.floor(self.video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)), + math.floor(self.video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + + @property + def resolution(self) -> CameraResolutionType: + return self._resolution + + def __enter__(self) -> RGBCamera: + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.video_capture.release() + + def intrinsics_matrix(self) -> CameraIntrinsicsMatrixType: + """Obtain the intrinsics matrix of the camera. + + Raises: + RuntimeError: You must explicitly pass an intrinsics object to the constructor. + + Returns: + CameraIntrinsicsMatrixType: The intrinsics matrix. + """ + if self._intrinsics_matrix is None: + raise RuntimeError( + "OpenCVVideoCapture does not have a preset intrinsics matrix. Pass it to the constructor if you know it." + ) + return self._intrinsics_matrix + + def _grab_images(self): + ret, image = self.video_capture.read() + if not ret: + raise RuntimeError("Can't receive frame (stream end?). Exiting...") + + self._frame = image + + def _retrieve_rgb_image(self) -> NumpyFloatImageType: + return ImageConverter.from_opencv_format(self._frame).image_in_numpy_format + + def _retrieve_rgb_image_as_int(self) -> NumpyIntImageType: + return ImageConverter.from_opencv_format(self._frame).image_in_numpy_int_format + + +if __name__ == "__main__": + camera = OpenCVVideoCapture() + + while True: + image = camera.get_rgb_image() + print(image.shape) + image = ImageConverter.from_numpy_format(image).image_in_opencv_format + + cv2.imshow("VideoCapture", image) + key = cv2.waitKey(10) + if key == ord("q"): + break diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv_camera.md b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv_camera.md new file mode 100644 index 00000000..1f7e773d --- /dev/null +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv_camera.md @@ -0,0 +1,6 @@ +# Generic OpenCV camera + +This `RGBCamera` implementation allows testing arbitrary cameras through the OpenCV `VideoCapture` interface. + +We currently do not support intrinsics calibration in airo-camera-toolkit. You can find the intrinsics of your camera +using [these instructions](https://docs.opencv.org/4.x/dc/dbb/tutorial_py_calibration.html). From bae757c446f2cd82177fc7d3fcf023a7c121767f Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 08:29:13 +0100 Subject: [PATCH 2/9] Modify changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 634070dc..bbb9f312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ This project uses a [CalVer](https://calver.org/) versioning scheme with monthly - Functions to convert from our numpy-based dataclass to and from open3d point clouds - `BoundingBox3DType` - `Zed2i.ULTRA_DEPTH_MODE` to enable the ultra depth setting for the Zed2i cameras - +- `OpenCVVideoCapture` implementation of `RGBCamera` for working with arbitrary cameras ### Changed From e01b5f997453f863f15bd0b47c118f6489915cdd Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 08:36:32 +0100 Subject: [PATCH 3/9] Add missing return type --- .../cameras/generic_opencv/generic_opencv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py index d21ebd26..44b88415 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py @@ -52,7 +52,7 @@ def intrinsics_matrix(self) -> CameraIntrinsicsMatrixType: ) return self._intrinsics_matrix - def _grab_images(self): + def _grab_images(self) -> None: ret, image = self.video_capture.read() if not ret: raise RuntimeError("Can't receive frame (stream end?). Exiting...") From b3e0048eb56bd29802c4c85e7e941d8fe28a504c Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 08:37:51 +0100 Subject: [PATCH 4/9] typo --- .../cameras/generic_opencv/generic_opencv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py index 44b88415..12875eee 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py @@ -10,7 +10,7 @@ class OpenCVVideoCapture(RGBCamera): - """Wrapper around OpenCV's VideoCapure so we can test the camera interface without external cameras.""" + """Wrapper around OpenCV's VideoCapture so we can test the camera interface without external cameras.""" def __init__( self, video_capture_args: Tuple[Any] = (0,), intrinsics_matrix: Optional[CameraIntrinsicsMatrixType] = None From 1d2175ed1f251a26a42c9cbf1c795fe57d38c110 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 08:44:00 +0100 Subject: [PATCH 5/9] Expand manual test --- .../cameras/generic_opencv/generic_opencv.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py index 12875eee..ad0fd71a 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py @@ -67,14 +67,24 @@ def _retrieve_rgb_image_as_int(self) -> NumpyIntImageType: if __name__ == "__main__": - camera = OpenCVVideoCapture() + import airo_camera_toolkit.cameras.manual_test_hw as test + import numpy as np + + camera = OpenCVVideoCapture(intrinsics_matrix=np.eye(3)) + + # Perform tests + test.manual_test_camera(camera) + test.manual_test_rgb_camera(camera) + test.profile_rgb_throughput(camera) + + # Live viewer + cv2.namedWindow("OpenCV Webcam RGB", cv2.WINDOW_NORMAL) while True: - image = camera.get_rgb_image() - print(image.shape) - image = ImageConverter.from_numpy_format(image).image_in_opencv_format + color_image = camera.get_rgb_image_as_int() + color_image = ImageConverter.from_numpy_int_format(color_image).image_in_opencv_format - cv2.imshow("VideoCapture", image) - key = cv2.waitKey(10) + cv2.imshow("OpenCV Webcam RGB", color_image) + key = cv2.waitKey(1) if key == ord("q"): break From 165d8569070a118fcd04caea1a0c4dd84011e70e Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 11:35:59 +0100 Subject: [PATCH 6/9] Rename generic_opencv --- airo-camera-toolkit/airo_camera_toolkit/cameras/README.md | 2 +- .../cameras/{generic_opencv => opencv_videocapture}/__init__.py | 0 .../opencv_videocapture.py} | 0 .../opencv_videocapture_camera.md} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename airo-camera-toolkit/airo_camera_toolkit/cameras/{generic_opencv => opencv_videocapture}/__init__.py (100%) rename airo-camera-toolkit/airo_camera_toolkit/cameras/{generic_opencv/generic_opencv.py => opencv_videocapture/opencv_videocapture.py} (100%) rename airo-camera-toolkit/airo_camera_toolkit/cameras/{generic_opencv/generic_opencv_camera.md => opencv_videocapture/opencv_videocapture_camera.md} (100%) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md b/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md index 8867e288..a09afe19 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/README.md @@ -7,7 +7,7 @@ This subpackage contains implementations of the camera interface for the cameras It also contains code to enable multiprocessed use of the camera streams: [multiprocessed camera](./multiprocess/) -There is also an implementation for generic RGB cameras using OpenCV `VideoCapture`: [generic OpenCV camera](./generic_opencv/) +There is also an implementation for generic RGB cameras using OpenCV `VideoCapture`: [OpenCV VideoCapture](./opencv_videocapture/) ## 1. Installation Implementations usually require the installation of SDKs, drivers etc. to communicate with the camera. diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/__init__.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/__init__.py similarity index 100% rename from airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/__init__.py rename to airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/__init__.py diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py similarity index 100% rename from airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv.py rename to airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv_camera.md b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture_camera.md similarity index 100% rename from airo-camera-toolkit/airo_camera_toolkit/cameras/generic_opencv/generic_opencv_camera.md rename to airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture_camera.md From b27c304fcfdbfb176bb47033c6cb3d3627b605da Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 11:52:00 +0100 Subject: [PATCH 7/9] Throw a more meaningful error when a video file was not found --- .../cameras/opencv_videocapture/opencv_videocapture.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py index ad0fd71a..abe7d131 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +import os from typing import Any, Optional, Tuple import cv2 @@ -16,8 +17,14 @@ def __init__( self, video_capture_args: Tuple[Any] = (0,), intrinsics_matrix: Optional[CameraIntrinsicsMatrixType] = None ) -> None: self.video_capture = cv2.VideoCapture(*video_capture_args) + + # If passing a video file, we want to check if it exists. Then, we can throw a more meaningful + # error if it does not. + if len(video_capture_args) > 0 and isinstance(video_capture_args[0], str): + if not os.path.isfile(video_capture_args[0]): + raise FileNotFoundError(f"Could not find video file {video_capture_args[0]}") if not self.video_capture.isOpened(): - raise RuntimeError("Cannot open camera") + raise RuntimeError(f"Cannot open camera {video_capture_args[0]}. Is it connected?") self.fps = self.video_capture.get(cv2.CAP_PROP_FPS) self._intrinsics_matrix = intrinsics_matrix From 976fc38dfa024cf149129883490f420a29b213a6 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Wed, 14 Feb 2024 13:08:25 +0100 Subject: [PATCH 8/9] Selectable resolution --- .../opencv_videocapture.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py index abe7d131..7d308400 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py @@ -13,8 +13,20 @@ class OpenCVVideoCapture(RGBCamera): """Wrapper around OpenCV's VideoCapture so we can test the camera interface without external cameras.""" + # Some standard resolutions that are likely to be supported by webcams. + # 16:9 + RESOLUTION_1080 = (1920, 1080) + RESOLUTION_720 = (1280, 720) + # 4:3 + RESOLUTION_768 = (1024, 768) + RESOLUTION_480 = (640, 480) + def __init__( - self, video_capture_args: Tuple[Any] = (0,), intrinsics_matrix: Optional[CameraIntrinsicsMatrixType] = None + self, + video_capture_args: Tuple[Any] = (0,), + intrinsics_matrix: Optional[CameraIntrinsicsMatrixType] = None, + resolution: CameraResolutionType = RESOLUTION_480, + fps: int = 30, ) -> None: self.video_capture = cv2.VideoCapture(*video_capture_args) @@ -26,9 +38,15 @@ def __init__( if not self.video_capture.isOpened(): raise RuntimeError(f"Cannot open camera {video_capture_args[0]}. Is it connected?") - self.fps = self.video_capture.get(cv2.CAP_PROP_FPS) + # Note that the following will not forcibly set the resolution. If the user's webcam + # does not support the desired resolution, OpenCV will silently select a close match. + self.video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, resolution[0]) + self.video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, resolution[1]) + self.video_capture.set(cv2.CAP_PROP_FPS, fps) + self._intrinsics_matrix = intrinsics_matrix + self.fps = self.video_capture.get(cv2.CAP_PROP_FPS) self._resolution = ( math.floor(self.video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)), math.floor(self.video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)), From 0f0157d71e20909863c7a486ca3b4e44920ec9f9 Mon Sep 17 00:00:00 2001 From: Mathieu De Coster Date: Thu, 22 Feb 2024 15:04:04 +0100 Subject: [PATCH 9/9] Raise EOFError at the end of a video stream --- .../cameras/opencv_videocapture/opencv_videocapture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py index 7d308400..af09b3e7 100644 --- a/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py +++ b/airo-camera-toolkit/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py @@ -79,8 +79,8 @@ def intrinsics_matrix(self) -> CameraIntrinsicsMatrixType: def _grab_images(self) -> None: ret, image = self.video_capture.read() - if not ret: - raise RuntimeError("Can't receive frame (stream end?). Exiting...") + if not ret: # When streaming a video, we will at some point reach the end. + raise EOFError("Can't receive frame (stream end?). Exiting...") self._frame = image