diff --git a/cpp/drafts/CMakeLists.txt b/cpp/drafts/CMakeLists.txt index 372f3235e..9d7cb86ed 100644 --- a/cpp/drafts/CMakeLists.txt +++ b/cpp/drafts/CMakeLists.txt @@ -11,7 +11,6 @@ endif() add_subdirectory(Compute) add_subdirectory(MatchPropagation) -add_subdirectory(NuScenes) add_subdirectory(Taskflow) # Disable for now because the ONNX runtime library was compiled with CUDA 11 and diff --git a/cpp/drafts/NuScenes/NuImages.hpp b/cpp/drafts/NuScenes/NuImages.hpp deleted file mode 100644 index da8e57217..000000000 --- a/cpp/drafts/NuScenes/NuImages.hpp +++ /dev/null @@ -1,198 +0,0 @@ -// ========================================================================== // -// This file is part of Sara, a basic set of libraries in C++ for computer -// vision. -// -// Copyright (C) 2021-present David Ok -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License v. 2.0. If a copy of the MPL was not distributed with this file, -// you can obtain one at http://mozilla.org/MPL/2.0/. -// ========================================================================== // - -//! @file - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - - -struct NuImages -{ - std::string version = "v1.0-mini"; - std::string dataroot = "/data/sets/nuscenes"; - bool verbose = true; - std::vector table_names = { - "category", "attribute", "visibility", - "instance", "sensor", "calibrated_sensor", - "ego_pose", "log", "scene", - "sample", "sample_data", "sample_annotation", - "map" // - }; - - using Token = std::string; - - struct Sample - { - Token log_token; - Token key_camera_token; - - //! @brief A sample is also referenced by a timestamp. - std::uint64_t timestamp; - }; - - // For a given sample, we index the images acquired from the 6 cameras - // (closest to this sample timestamp). - struct SampleData - { - Token prev; - Token next; - - Token sample_token; - Token ego_pose_token; - Token calibrated_sensor_token; - - std::uint64_t timestamp; - std::string fileformat; - bool is_key_frame; - int width; - int height; - - //! @brief An image file, lidar point cloud file or radar velocity file. - std::string filename; - }; - - struct ObjectAnnotation - { - Token sample_data_token; - Token category_token; - std::vector attribute_tokens; - - // The 2D bounding box. - Eigen::Vector4f bbox; - - // TODO: mask. - }; - - struct Category - { - std::string name; - std::string description; - }; - - struct EgoPose - { - Eigen::Quaternionf rotation; - Eigen::Vector3f translation; - - //! IMU data. - //! - //! @brief Gyroscope. - Eigen::Vector3f rotation_rate; - //! @brief Acceleration. - Eigen::Vector3f acceleration; - - float speed; - - //! @brief A sample is also referenced by a timestamp. - std::uint64_t timestamp; - }; - - struct CalibratedSensor - { - // This is what I think it is. - struct CameraDistortion - { - Eigen::Vector3f k; - Eigen::Vector2f p; - }; - - Eigen::Quaternionf rotation; - Eigen::Vector3f translation; - Eigen::Matrix3f calibration_matrix; - CameraDistortion camera_distortion; - - Token sensor_token; - }; - - struct Attribute - { - std::string name; - std::string description; - }; - - struct Sensor - { - std::string channel; - std::string modality; - }; - - struct Log - { - std::string logfile; - std::string location; - std::string vehicle; - std::string date_captured; - }; - - std::unordered_map sample_table; - std::unordered_map sample_data_table; - std::unordered_map object_annotation_table; - std::unordered_map category_table; - std::unordered_map ego_pose_table; - std::unordered_map calibrated_sensor_table; - std::unordered_map attribute_table; - std::unordered_map sensor_table; - std::unordered_map log_table; - - NuImages(const std::string& version_, // - const std::string& dataroot_, // - bool verbose_/*, // - float map_resolution = 0.1*/); - - auto load_sample_table() -> void; - - auto load_sample_data_table() -> void; - - auto load_object_annotation_table() -> void; - - auto load_category_table() -> void; - - auto load_ego_pose_table() -> void; - - auto load_calibrated_sensor_table() -> void; - - auto load_attribute_table() -> void; - - auto load_sensor_table() -> void; - - auto load_log_table() -> void; - - template - auto filter_by_sample_token(const std::unordered_map& table, const Token& value) const - { - auto rows = std::vector{}; - std::for_each(table.cbegin(), table.cend(), [&](const auto& row) { - if (row.second.sample_token == value) - rows.emplace_back(row.second); - }); - return rows; - } - - auto get_data_path(const SampleData& data) const -> std::string; - - auto get_ego_pose(const SampleData& data) const -> const EgoPose&; - - auto get_calibrated_sensor(const SampleData& data) const - -> const CalibratedSensor&; - - auto get_sensor(const CalibratedSensor& calibrated_sensor) const - -> const Sensor&; -}; diff --git a/cpp/drafts/NuScenes/NuScenes.hpp b/cpp/drafts/NuScenes/NuScenes.hpp deleted file mode 100644 index ad7b9a697..000000000 --- a/cpp/drafts/NuScenes/NuScenes.hpp +++ /dev/null @@ -1,223 +0,0 @@ -// ========================================================================== // -// This file is part of Sara, a basic set of libraries in C++ for computer -// vision. -// -// Copyright (C) 2021-present David Ok -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License v. 2.0. If a copy of the MPL was not distributed with this file, -// you can obtain one at http://mozilla.org/MPL/2.0/. -// ========================================================================== // - -//! @file - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include - - -/*! - * The typical pattern regarding the data organization in NuScenes tables is as - * follows: - * - Each row of a any table is referenced by a UUID termed as "token" in - * NuScenes terminology. - * From a data structure point of view, each row can be stored as a hash - * table. - * - Each table can also be viewed as a doubly-linked list because of "prev" - * and "next" fields. - */ -struct NuScenes -{ - std::string version = "v1.0-mini"; - std::string dataroot = "/data/sets/nuscenes"; - bool verbose = true; - std::vector table_names = { - "category", "attribute", "visibility", - "instance", "sensor", "calibrated_sensor", - "ego_pose", "log", "scene", - "sample", "sample_data", "sample_annotation", - "map" // - }; - - using Token = std::string; - - struct Sample - { - Token prev; - Token next; - - //! @brief The scene in which the sample is acquired. - Token scene_token; - - //! @brief A sample is also referenced by a timestamp. - std::uint64_t timestamp; - }; - - // For a given sample, we index the following data: - // - images acquired from the 6 cameras (closest to this sample timestamp). - // - point cloud acquired from the lidar device (closest to this sample - // timestamp), - // - velocities acquired from the radar device (closest to this sample - // timestamp) - struct SampleData - { - Token prev; - Token next; - - Token sample_token; - Token ego_pose_token; - Token calibrated_sensor_token; - - std::uint64_t timestamp; - std::string fileformat; - bool is_key_frame; - std::optional width; - std::optional height; - - //! @brief An image file, lidar point cloud file or radar velocity file. - std::string filename; - - auto is_image() const - { - return width.has_value() && height.has_value() && fileformat == "jpg"; - } - }; - - struct SampleAnnotation - { - Token prev; - Token next; - - Token sample_token; - Token instance_token; - int visibility_token; - std::vector attributes_tokens; - - // The 3D bounding box. - Eigen::Vector3f size; - Eigen::Vector3f translation; - Eigen::Quaternionf rotation; - - // The lidar and radar metadata. - int num_lidar_pts; - int num_radar_pts; - }; - - struct Category - { - std::string name; - std::string description; - int index; - }; - - struct EgoPose - { - Eigen::Quaternionf rotation; - Eigen::Vector3f translation; - - //! @brief A sample is also referenced by a timestamp. - std::uint64_t timestamp; - }; - - struct CalibratedSensor - { - Eigen::Quaternionf rotation; - Eigen::Vector3f translation; - std::optional calibration_matrix; - Token sensor_token; - }; - - // An object instance should be understood as the same object that is seen - // multiple times in multiple image frames, or point cloud data and sensor - // data. - struct Instance - { - Token category_token; - int number_of_annotations; - Token first_annotation_token; - Token last_annotation_token; - }; - - struct Visibility - { - std::string description; - std::string level; - }; - - struct Attribute - { - std::string name; - std::string description; - }; - - struct Sensor - { - std::string channel; - std::string modality; - }; - - std::unordered_map sample_table; - std::unordered_map sample_data_table; - std::unordered_map sample_annotation_table; - std::unordered_map category_table; - std::unordered_map ego_pose_table; - std::unordered_map calibrated_sensor_table; - std::unordered_map instance_table; - std::unordered_map visibility_table; - std::unordered_map attribute_table; - std::unordered_map sensor_table; - - NuScenes(const std::string& version_, // - const std::string& dataroot_, // - bool verbose_/*, // - float map_resolution = 0.1*/); - - auto load_sample_table() -> void; - - auto load_sample_data_table() -> void; - - auto load_sample_annotation_table() -> void; - - auto load_category_table() -> void; - - auto load_ego_pose_table() -> void; - - auto load_calibrated_sensor_table() -> void; - - auto load_instance_table() -> void; - - auto load_visibility_table() -> void; - - auto load_attribute_table() -> void; - - auto load_sensor_table() -> void; - - template - auto filter_by_sample_token(const std::unordered_map& table, const Token& value) const - { - auto rows = std::vector{}; - std::for_each(table.cbegin(), table.cend(), [&](const auto& row) { - if (row.second.sample_token == value) - rows.emplace_back(row.second); - }); - return rows; - } - - auto get_data_path(const SampleData& data) const -> std::string; - - auto get_ego_pose(const SampleData& data) const -> const EgoPose&; - - auto get_calibrated_sensor(const SampleData& data) const - -> const CalibratedSensor&; - - auto get_sensor(const CalibratedSensor& calibrated_sensor) const - -> const Sensor&; -}; diff --git a/cpp/examples/Kalpana/Qt/kalpana_hello_camera.cpp b/cpp/examples/Kalpana/Qt/kalpana_hello_camera.cpp index 8ff6696cc..b3247c815 100644 --- a/cpp/examples/Kalpana/Qt/kalpana_hello_camera.cpp +++ b/cpp/examples/Kalpana/Qt/kalpana_hello_camera.cpp @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/cpp/examples/Sara/CMakeLists.txt b/cpp/examples/Sara/CMakeLists.txt index eba9d237b..f12b64101 100644 --- a/cpp/examples/Sara/CMakeLists.txt +++ b/cpp/examples/Sara/CMakeLists.txt @@ -13,5 +13,6 @@ if (SARA_BUILD_VIDEOIO) add_subdirectory(VideoIO) endif () +add_subdirectory(Datasets) add_subdirectory(ChessboardDetection) add_subdirectory(NeuralNetworks) diff --git a/cpp/drafts/NuScenes/CMakeLists.txt b/cpp/examples/Sara/Datasets/CMakeLists.txt similarity index 68% rename from cpp/drafts/NuScenes/CMakeLists.txt rename to cpp/examples/Sara/Datasets/CMakeLists.txt index e90f7edba..fa812793a 100644 --- a/cpp/drafts/NuScenes/CMakeLists.txt +++ b/cpp/examples/Sara/Datasets/CMakeLists.txt @@ -1,15 +1,8 @@ -add_library(NuScenes STATIC NuImages.hpp NuImages.cpp NuScenes.hpp NuScenes.cpp) -target_include_directories( - NuScenes # - PUBLIC ${CMAKE_SOURCE_DIR}/cpp/third-party) -target_link_libraries(NuScenes PUBLIC Eigen3::Eigen) -set_property(TARGET NuScenes PROPERTY FOLDER "Datasets/NuScenes") - add_executable(nuimages_cpp_devkit_usage_example nuimages_cpp_devkit_usage_example.cpp) target_link_libraries( nuimages_cpp_devkit_usage_example - PRIVATE NuScenes # + PRIVATE DO::Sara::NuScenes # DO::Sara::Core # DO::Sara::Graphics # DO::Sara::ImageIO) @@ -20,7 +13,7 @@ add_executable(nuscenes_cpp_devkit_usage_example nuscenes_cpp_devkit_usage_example.cpp) target_link_libraries( nuscenes_cpp_devkit_usage_example - PRIVATE NuScenes # + PRIVATE DO::Sara::NuScenes # DO::Sara::Core # DO::Sara::Graphics # DO::Sara::ImageIO) diff --git a/cpp/drafts/NuScenes/nuimages_cpp_devkit_usage_example.cpp b/cpp/examples/Sara/Datasets/nuimages_cpp_devkit_usage_example.cpp similarity index 85% rename from cpp/drafts/NuScenes/nuimages_cpp_devkit_usage_example.cpp rename to cpp/examples/Sara/Datasets/nuimages_cpp_devkit_usage_example.cpp index 9defc1394..b9d50f488 100644 --- a/cpp/drafts/NuScenes/nuimages_cpp_devkit_usage_example.cpp +++ b/cpp/examples/Sara/Datasets/nuimages_cpp_devkit_usage_example.cpp @@ -11,11 +11,10 @@ //! @file +#include #include #include -#include - namespace sara = DO::Sara; @@ -26,9 +25,8 @@ GRAPHICS_MAIN() const auto nuimages_version = "v1.0-mini"s; const auto nuimages_root_path = "/Users/david/Downloads/nuimages-v1.0-mini"s; - const auto nuimages = NuImages{nuimages_version, nuimages_root_path, true}; - - + const auto nuimages = + sara::NuImages{nuimages_version, nuimages_root_path, true}; return 0; } diff --git a/cpp/drafts/NuScenes/nuscenes_cpp_devkit_usage_example.cpp b/cpp/examples/Sara/Datasets/nuscenes_cpp_devkit_usage_example.cpp similarity index 93% rename from cpp/drafts/NuScenes/nuscenes_cpp_devkit_usage_example.cpp rename to cpp/examples/Sara/Datasets/nuscenes_cpp_devkit_usage_example.cpp index dce0747a6..1c31591b3 100644 --- a/cpp/drafts/NuScenes/nuscenes_cpp_devkit_usage_example.cpp +++ b/cpp/examples/Sara/Datasets/nuscenes_cpp_devkit_usage_example.cpp @@ -11,11 +11,10 @@ //! @file +#include #include #include -#include - namespace sara = DO::Sara; @@ -31,7 +30,7 @@ namespace sara = DO::Sara; // In the world reference, the car is: // - at the following coordinates given by 'ego_pose.translation' // - oriented by the quaternion 'ego_pose.rotation' -auto local_to_world_transform(const NuScenes::EgoPose& ego_pose) +auto local_to_world_transform(const sara::NuScenes::EgoPose& ego_pose) { // return rigid_body_transform(ego_pose.rotation, ego_pose.translation) @@ -48,7 +47,7 @@ auto local_to_world_transform(const NuScenes::EgoPose& ego_pose) return T; } -auto world_to_local_transform(const NuScenes::EgoPose& ego_pose) +auto world_to_local_transform(const sara::NuScenes::EgoPose& ego_pose) { // return rigid_body_transform(ego_pose.rotation, ego_pose.translation) // .inverse(); @@ -67,7 +66,7 @@ auto world_to_local_transform(const NuScenes::EgoPose& ego_pose) } auto local_to_sensor_transform( - const NuScenes::CalibratedSensor& calibrated_sensor) + const sara::NuScenes::CalibratedSensor& calibrated_sensor) { const Eigen::Matrix3f Rt = calibrated_sensor // .rotation // @@ -111,11 +110,13 @@ struct Box // The coordinates follow the automotive axis convention. auto vertices = Eigen::Matrix{}; + // clang-format off vertices << // Back face | Front face +1, +1, +1, +1, -1, -1, -1, -1, // +1, -1, -1, +1, +1, -1, -1, +1, // +1, +1, -1, -1, +1, +1, -1, -1; // + // clang-format on // The first 4 vertices are the back face, enumerated in the following // order: @@ -148,8 +149,8 @@ struct Box return vertices; } - auto draw(const NuScenes::EgoPose& ego_pose, - const NuScenes::CalibratedSensor& calibrated_sensor) const + auto draw(const sara::NuScenes::EgoPose& ego_pose, + const sara::NuScenes::CalibratedSensor& calibrated_sensor) const { const Eigen::Matrix4f T = local_to_sensor_transform(calibrated_sensor) * world_to_local_transform(ego_pose); @@ -207,7 +208,8 @@ GRAPHICS_MAIN() const auto nuscenes_version = "v1.0-mini"s; const auto nuscenes_root_path = "/home/david/Downloads/nuscenes"s; - const auto nuscenes = NuScenes{nuscenes_version, nuscenes_root_path, true}; + const auto nuscenes = + sara::NuScenes{nuscenes_version, nuscenes_root_path, true}; for (const auto& [sample_token, sample] : nuscenes.sample_table) { @@ -312,10 +314,9 @@ GRAPHICS_MAIN() const auto& category = nuscenes.category_table.at(instance.category_token); - const auto box = Box{annotation.size, - annotation.rotation.toRotationMatrix(), - annotation.translation, - category.name}; + const auto box = + Box{annotation.size, annotation.rotation.toRotationMatrix(), + annotation.translation, category.name}; box.draw(ego_pose, calibrated_sensor); } diff --git a/cpp/examples/Sara/MultiViewGeometry/essential_5_point_example.cpp b/cpp/examples/Sara/MultiViewGeometry/essential_5_point_example.cpp index d61d4a194..c83d01885 100644 --- a/cpp/examples/Sara/MultiViewGeometry/essential_5_point_example.cpp +++ b/cpp/examples/Sara/MultiViewGeometry/essential_5_point_example.cpp @@ -13,6 +13,7 @@ //! This program parses Strecha's datasets. #include +#include #include #include #include diff --git a/cpp/examples/Sara/MultiViewGeometry/relative_pose_estimation_example.cpp b/cpp/examples/Sara/MultiViewGeometry/relative_pose_estimation_example.cpp index 00eddd5a5..cda189ea2 100644 --- a/cpp/examples/Sara/MultiViewGeometry/relative_pose_estimation_example.cpp +++ b/cpp/examples/Sara/MultiViewGeometry/relative_pose_estimation_example.cpp @@ -20,6 +20,7 @@ //! without worrying whether the triangulated points are cheiral just works //! better. Then after the pose recovery at the end just gives better results. +#include #include #include #include diff --git a/cpp/examples/Sara/MultiViewGeometry/two_view_bundle_adjustment_example.cpp b/cpp/examples/Sara/MultiViewGeometry/two_view_bundle_adjustment_example.cpp index 70de5090c..505e113e9 100644 --- a/cpp/examples/Sara/MultiViewGeometry/two_view_bundle_adjustment_example.cpp +++ b/cpp/examples/Sara/MultiViewGeometry/two_view_bundle_adjustment_example.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/cpp/src/DO/Sara/CMakeLists.txt b/cpp/src/DO/Sara/CMakeLists.txt index 92d68a74b..7e033c1e3 100644 --- a/cpp/src/DO/Sara/CMakeLists.txt +++ b/cpp/src/DO/Sara/CMakeLists.txt @@ -32,6 +32,8 @@ include(UseDOSaraRANSAC) include(UseDOSaraVisualization) +add_subdirectory(Datasets) + add_subdirectory(ChessboardDetection) add_subdirectory(Calibration) diff --git a/cpp/src/DO/Sara/Datasets/CMakeLists.txt b/cpp/src/DO/Sara/Datasets/CMakeLists.txt new file mode 100644 index 000000000..4f350d618 --- /dev/null +++ b/cpp/src/DO/Sara/Datasets/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(NuScenes) diff --git a/cpp/src/DO/Sara/Datasets/NuScenes/CMakeLists.txt b/cpp/src/DO/Sara/Datasets/NuScenes/CMakeLists.txt new file mode 100644 index 000000000..5810714ce --- /dev/null +++ b/cpp/src/DO/Sara/Datasets/NuScenes/CMakeLists.txt @@ -0,0 +1,12 @@ +add_library( + DO_Sara_NuScenes # + STATIC # + NuImages.hpp NuImages.cpp # + NuScenes.hpp NuScenes.cpp) +add_library(DO::Sara::NuScenes ALIAS DO_Sara_NuScenes) +target_include_directories( + DO_Sara_NuScenes # + PUBLIC ${CMAKE_SOURCE_DIR}/cpp/src # + ${CMAKE_SOURCE_DIR}/cpp/third-party) +target_link_libraries(DO_Sara_NuScenes PUBLIC Eigen3::Eigen) +set_property(TARGET DO_Sara_NuScenes PROPERTY FOLDER "Libraries/Datasets/NuScenes") diff --git a/cpp/drafts/NuScenes/NuImages.cpp b/cpp/src/DO/Sara/Datasets/NuScenes/NuImages.cpp similarity index 98% rename from cpp/drafts/NuScenes/NuImages.cpp rename to cpp/src/DO/Sara/Datasets/NuScenes/NuImages.cpp index e36537897..c323f3b34 100644 --- a/cpp/drafts/NuScenes/NuImages.cpp +++ b/cpp/src/DO/Sara/Datasets/NuScenes/NuImages.cpp @@ -9,7 +9,7 @@ // you can obtain one at http://mozilla.org/MPL/2.0/. // ========================================================================== // -#include +#include #include @@ -18,6 +18,8 @@ namespace fs = std::filesystem; +using namespace DO::Sara; + static auto load_json(const std::string& dataroot, const std::string& version, const std::string& table_name) -> nlohmann::json diff --git a/cpp/src/DO/Sara/Datasets/NuScenes/NuImages.hpp b/cpp/src/DO/Sara/Datasets/NuScenes/NuImages.hpp new file mode 100644 index 000000000..6fcfb1945 --- /dev/null +++ b/cpp/src/DO/Sara/Datasets/NuScenes/NuImages.hpp @@ -0,0 +1,203 @@ +// ========================================================================== // +// This file is part of Sara, a basic set of libraries in C++ for computer +// vision. +// +// Copyright (C) 2021-present David Ok +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License v. 2.0. If a copy of the MPL was not distributed with this file, +// you can obtain one at http://mozilla.org/MPL/2.0/. +// ========================================================================== // + +//! @file + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace DO::Sara { + + struct NuImages + { + std::string version = "v1.0-mini"; + std::string dataroot = "/data/sets/nuscenes"; + bool verbose = true; + std::vector table_names = { + "category", "attribute", "visibility", + "instance", "sensor", "calibrated_sensor", + "ego_pose", "log", "scene", + "sample", "sample_data", "sample_annotation", + "map" // + }; + + using Token = std::string; + + struct Sample + { + Token log_token; + Token key_camera_token; + + //! @brief A sample is also referenced by a timestamp. + std::uint64_t timestamp; + }; + + // For a given sample, we index the images acquired from the 6 cameras + // (closest to this sample timestamp). + struct SampleData + { + Token prev; + Token next; + + Token sample_token; + Token ego_pose_token; + Token calibrated_sensor_token; + + std::uint64_t timestamp; + std::string fileformat; + bool is_key_frame; + int width; + int height; + + //! @brief An image file, lidar point cloud file or radar velocity file. + std::string filename; + }; + + struct ObjectAnnotation + { + Token sample_data_token; + Token category_token; + std::vector attribute_tokens; + + // The 2D bounding box. + Eigen::Vector4f bbox; + + // TODO: mask. + }; + + struct Category + { + std::string name; + std::string description; + }; + + struct EgoPose + { + Eigen::Quaternionf rotation; + Eigen::Vector3f translation; + + //! IMU data. + //! + //! @brief Gyroscope. + Eigen::Vector3f rotation_rate; + //! @brief Acceleration. + Eigen::Vector3f acceleration; + + float speed; + + //! @brief A sample is also referenced by a timestamp. + std::uint64_t timestamp; + }; + + struct CalibratedSensor + { + // This is what I think it is. + struct CameraDistortion + { + Eigen::Vector3f k; + Eigen::Vector2f p; + }; + + Eigen::Quaternionf rotation; + Eigen::Vector3f translation; + Eigen::Matrix3f calibration_matrix; + CameraDistortion camera_distortion; + + Token sensor_token; + }; + + struct Attribute + { + std::string name; + std::string description; + }; + + struct Sensor + { + std::string channel; + std::string modality; + }; + + struct Log + { + std::string logfile; + std::string location; + std::string vehicle; + std::string date_captured; + }; + + std::unordered_map sample_table; + std::unordered_map sample_data_table; + std::unordered_map object_annotation_table; + std::unordered_map category_table; + std::unordered_map ego_pose_table; + std::unordered_map calibrated_sensor_table; + std::unordered_map attribute_table; + std::unordered_map sensor_table; + std::unordered_map log_table; + + NuImages(const std::string& version_, // + const std::string& dataroot_, // + bool verbose_/*, // + float map_resolution = 0.1*/); + + auto load_sample_table() -> void; + + auto load_sample_data_table() -> void; + + auto load_object_annotation_table() -> void; + + auto load_category_table() -> void; + + auto load_ego_pose_table() -> void; + + auto load_calibrated_sensor_table() -> void; + + auto load_attribute_table() -> void; + + auto load_sensor_table() -> void; + + auto load_log_table() -> void; + + template + auto filter_by_sample_token(const std::unordered_map& table, + const Token& value) const + { + auto rows = std::vector{}; + std::for_each(table.cbegin(), table.cend(), [&](const auto& row) { + if (row.second.sample_token == value) + rows.emplace_back(row.second); + }); + return rows; + } + + auto get_data_path(const SampleData& data) const -> std::string; + + auto get_ego_pose(const SampleData& data) const -> const EgoPose&; + + auto get_calibrated_sensor(const SampleData& data) const + -> const CalibratedSensor&; + + auto get_sensor(const CalibratedSensor& calibrated_sensor) const + -> const Sensor&; + }; + +} // namespace DO::Sara diff --git a/cpp/drafts/NuScenes/NuScenes.cpp b/cpp/src/DO/Sara/Datasets/NuScenes/NuScenes.cpp similarity index 99% rename from cpp/drafts/NuScenes/NuScenes.cpp rename to cpp/src/DO/Sara/Datasets/NuScenes/NuScenes.cpp index 55894e3d1..cab1af4f0 100644 --- a/cpp/drafts/NuScenes/NuScenes.cpp +++ b/cpp/src/DO/Sara/Datasets/NuScenes/NuScenes.cpp @@ -9,7 +9,7 @@ // you can obtain one at http://mozilla.org/MPL/2.0/. // ========================================================================== // -#include +#include #include @@ -18,6 +18,8 @@ namespace fs = std::filesystem; +using namespace DO::Sara; + static auto load_json(const std::string& dataroot, const std::string& version, const std::string& table_name) -> nlohmann::json diff --git a/cpp/src/DO/Sara/Datasets/NuScenes/NuScenes.hpp b/cpp/src/DO/Sara/Datasets/NuScenes/NuScenes.hpp new file mode 100644 index 000000000..0c782782c --- /dev/null +++ b/cpp/src/DO/Sara/Datasets/NuScenes/NuScenes.hpp @@ -0,0 +1,227 @@ +// ========================================================================== // +// This file is part of Sara, a basic set of libraries in C++ for computer +// vision. +// +// Copyright (C) 2021-present David Ok +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License v. 2.0. If a copy of the MPL was not distributed with this file, +// you can obtain one at http://mozilla.org/MPL/2.0/. +// ========================================================================== // + +//! @file + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DO::Sara { + /*! + * The typical pattern regarding the data organization in NuScenes tables is + * as follows: + * - Each row of a any table is referenced by a UUID termed as "token" in + * NuScenes terminology. + * From a data structure point of view, each row can be stored as a hash + * table. + * - Each table can also be viewed as a doubly-linked list because of "prev" + * and "next" fields. + */ + struct NuScenes + { + std::string version = "v1.0-mini"; + std::string dataroot = "/data/sets/nuscenes"; + bool verbose = true; + std::vector table_names = { + "category", "attribute", "visibility", + "instance", "sensor", "calibrated_sensor", + "ego_pose", "log", "scene", + "sample", "sample_data", "sample_annotation", + "map" // + }; + + using Token = std::string; + + struct Sample + { + Token prev; + Token next; + + //! @brief The scene in which the sample is acquired. + Token scene_token; + + //! @brief A sample is also referenced by a timestamp. + std::uint64_t timestamp; + }; + + // For a given sample, we index the following data: + // - images acquired from the 6 cameras (closest to this sample timestamp). + // - point cloud acquired from the lidar device (closest to this sample + // timestamp), + // - velocities acquired from the radar device (closest to this sample + // timestamp) + struct SampleData + { + Token prev; + Token next; + + Token sample_token; + Token ego_pose_token; + Token calibrated_sensor_token; + + std::uint64_t timestamp; + std::string fileformat; + bool is_key_frame; + std::optional width; + std::optional height; + + //! @brief An image file, lidar point cloud file or radar velocity file. + std::string filename; + + auto is_image() const + { + return width.has_value() && height.has_value() && fileformat == "jpg"; + } + }; + + struct SampleAnnotation + { + Token prev; + Token next; + + Token sample_token; + Token instance_token; + int visibility_token; + std::vector attributes_tokens; + + // The 3D bounding box. + Eigen::Vector3f size; + Eigen::Vector3f translation; + Eigen::Quaternionf rotation; + + // The lidar and radar metadata. + int num_lidar_pts; + int num_radar_pts; + }; + + struct Category + { + std::string name; + std::string description; + int index; + }; + + struct EgoPose + { + Eigen::Quaternionf rotation; + Eigen::Vector3f translation; + + //! @brief A sample is also referenced by a timestamp. + std::uint64_t timestamp; + }; + + struct CalibratedSensor + { + Eigen::Quaternionf rotation; + Eigen::Vector3f translation; + std::optional calibration_matrix; + Token sensor_token; + }; + + // An object instance should be understood as the same object that is seen + // multiple times in multiple image frames, or point cloud data and sensor + // data. + struct Instance + { + Token category_token; + int number_of_annotations; + Token first_annotation_token; + Token last_annotation_token; + }; + + struct Visibility + { + std::string description; + std::string level; + }; + + struct Attribute + { + std::string name; + std::string description; + }; + + struct Sensor + { + std::string channel; + std::string modality; + }; + + std::unordered_map sample_table; + std::unordered_map sample_data_table; + std::unordered_map sample_annotation_table; + std::unordered_map category_table; + std::unordered_map ego_pose_table; + std::unordered_map calibrated_sensor_table; + std::unordered_map instance_table; + std::unordered_map visibility_table; + std::unordered_map attribute_table; + std::unordered_map sensor_table; + + NuScenes(const std::string& version_, // + const std::string& dataroot_, // + bool verbose_/*, // + float map_resolution = 0.1*/); + + auto load_sample_table() -> void; + + auto load_sample_data_table() -> void; + + auto load_sample_annotation_table() -> void; + + auto load_category_table() -> void; + + auto load_ego_pose_table() -> void; + + auto load_calibrated_sensor_table() -> void; + + auto load_instance_table() -> void; + + auto load_visibility_table() -> void; + + auto load_attribute_table() -> void; + + auto load_sensor_table() -> void; + + template + auto filter_by_sample_token(const std::unordered_map& table, + const Token& value) const + { + auto rows = std::vector{}; + std::for_each(table.cbegin(), table.cend(), [&](const auto& row) { + if (row.second.sample_token == value) + rows.emplace_back(row.second); + }); + return rows; + } + + auto get_data_path(const SampleData& data) const -> std::string; + + auto get_ego_pose(const SampleData& data) const -> const EgoPose&; + + auto get_calibrated_sensor(const SampleData& data) const + -> const CalibratedSensor&; + + auto get_sensor(const CalibratedSensor& calibrated_sensor) const + -> const Sensor&; + }; + +} // namespace DO::Sara diff --git a/cpp/src/DO/Sara/MultiViewGeometry/Datasets/Strecha.hpp b/cpp/src/DO/Sara/Datasets/Strecha/Utilities.hpp similarity index 100% rename from cpp/src/DO/Sara/MultiViewGeometry/Datasets/Strecha.hpp rename to cpp/src/DO/Sara/Datasets/Strecha/Utilities.hpp diff --git a/cpp/src/DO/Sara/MultiViewGeometry.hpp b/cpp/src/DO/Sara/MultiViewGeometry.hpp index 909a705c8..8c0240784 100644 --- a/cpp/src/DO/Sara/MultiViewGeometry.hpp +++ b/cpp/src/DO/Sara/MultiViewGeometry.hpp @@ -16,8 +16,6 @@ #include #include -#include - #include #include #include