From f3c8ab89c14441324721cd86e2dd34cc3f75f564 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:04:28 +0900 Subject: [PATCH 01/52] refactor(openscenario_interpreter): delete ros messages from basic types --- .../include/openscenario_interpreter/syntax/double.hpp | 7 ++++--- .../include/openscenario_interpreter/syntax/integer.hpp | 9 +++++---- .../openscenario_interpreter/syntax/unsigned_integer.hpp | 7 ++++--- .../openscenario_interpreter/syntax/unsigned_short.hpp | 7 ++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/double.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/double.hpp index 13b02b4caac..747d48ccac5 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/double.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/double.hpp @@ -15,16 +15,17 @@ #ifndef OPENSCENARIO_INTERPRETER__SYNTAX__DOUBLE_HPP_ #define OPENSCENARIO_INTERPRETER__SYNTAX__DOUBLE_HPP_ -#include #include namespace openscenario_interpreter { inline namespace syntax { -struct Double : public std_msgs::msg::Float64 +struct Double { - using value_type = decltype(std_msgs::msg::Float64::data); + using value_type = double; + + value_type data = 0.0; Double() = default; diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp index 9c090abe082..afaf8f073d0 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp @@ -15,18 +15,19 @@ #ifndef OPENSCENARIO_INTERPRETER__SYNTAX__INTEGER_HPP_ #define OPENSCENARIO_INTERPRETER__SYNTAX__INTEGER_HPP_ -#include #include -#include +#include #include namespace openscenario_interpreter { inline namespace syntax { -struct Integer : public std_msgs::msg::Int64 +struct Integer { - using value_type = decltype(std_msgs::msg::Int64::data); + using value_type = std::int64_t; + + value_type data = 0; explicit Integer() = default; diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_integer.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_integer.hpp index fe2e8cf51eb..e0a73d29f09 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_integer.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_integer.hpp @@ -17,16 +17,17 @@ #include #include -#include #include namespace openscenario_interpreter { inline namespace syntax { -struct UnsignedInteger : public std_msgs::msg::UInt64 +struct UnsignedInteger { - using value_type = decltype(std_msgs::msg::UInt64::data); + using value_type = std::uint64_t; + + value_type data; explicit UnsignedInteger(value_type value = {}); diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_short.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_short.hpp index 3f9c5f57a91..73b4bc34455 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_short.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/unsigned_short.hpp @@ -16,16 +16,17 @@ #define OPENSCENARIO_INTERPRETER__SYNTAX__UNSIGNED_SHORT_HPP_ #include -#include #include namespace openscenario_interpreter { inline namespace syntax { -struct UnsignedShort : public std_msgs::msg::UInt16 +struct UnsignedShort { - using value_type = decltype(std_msgs::msg::UInt16::data); + using value_type = std::uint16_t; + + value_type data = 0U; explicit UnsignedShort() = default; From 6332613a2c78b033e217d525c31ff5be2b1fa9d1 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:06:41 +0900 Subject: [PATCH 02/52] refactor(openscenario_interpreter): comment-out UserDefinedDistribution --- .../deterministic_single_parameter_distribution_type.hpp | 1 - .../syntax/stochastic_distribution_type.hpp | 1 - .../deterministic_single_parameter_distribution_type.cpp | 4 ++-- .../src/syntax/stochastic_distribution_type.cpp | 4 ++-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/deterministic_single_parameter_distribution_type.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/deterministic_single_parameter_distribution_type.hpp index 1b091c5807e..21995aeb99e 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/deterministic_single_parameter_distribution_type.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/deterministic_single_parameter_distribution_type.hpp @@ -19,7 +19,6 @@ #include #include #include -#include #include namespace openscenario_interpreter diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/stochastic_distribution_type.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/stochastic_distribution_type.hpp index 0761e1e78bf..6b73fd4184f 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/stochastic_distribution_type.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/stochastic_distribution_type.hpp @@ -21,7 +21,6 @@ #include #include #include -#include #include namespace openscenario_interpreter diff --git a/openscenario/openscenario_interpreter/src/syntax/deterministic_single_parameter_distribution_type.cpp b/openscenario/openscenario_interpreter/src/syntax/deterministic_single_parameter_distribution_type.cpp index cce7e14edac..c2cb7d147ef 100644 --- a/openscenario/openscenario_interpreter/src/syntax/deterministic_single_parameter_distribution_type.cpp +++ b/openscenario/openscenario_interpreter/src/syntax/deterministic_single_parameter_distribution_type.cpp @@ -25,8 +25,8 @@ DeterministicSingleParameterDistributionType::DeterministicSingleParameterDistri : Group( choice(tree, std::make_pair("DistributionSet", [&](auto && node){ return make(node, scope);}), - std::make_pair("DistributionRange", [&](auto && node){ return make(node, scope);}), - std::make_pair("UserDefinedDistribution", [&](auto && node){ return make(node, scope);}))) + std::make_pair("DistributionRange", [&](auto && node){ return make(node, scope);}))) +// std::make_pair("UserDefinedDistribution", [&](auto && node){ return make(node, scope);}))) // clang-format on { } diff --git a/openscenario/openscenario_interpreter/src/syntax/stochastic_distribution_type.cpp b/openscenario/openscenario_interpreter/src/syntax/stochastic_distribution_type.cpp index 4f2d334ae5b..317166aa782 100644 --- a/openscenario/openscenario_interpreter/src/syntax/stochastic_distribution_type.cpp +++ b/openscenario/openscenario_interpreter/src/syntax/stochastic_distribution_type.cpp @@ -27,8 +27,8 @@ StochasticDistributionType::StochasticDistributionType(const pugi::xml_node & no std::make_pair("NormalDistribution", [&](auto && node){return make(node, scope);}), std::make_pair("UniformDistribution", [&](auto && node){return make(node, scope);}), std::make_pair("PoissonDistribution", [&](auto && node){return make(node, scope);}), - std::make_pair("Histogram", [&](auto && node){return make(node, scope);}), - std::make_pair("UserDefinedDistribution", [&](auto && node){return make(node, scope);}))) + std::make_pair("Histogram", [&](auto && node){return make(node, scope);}))) +// std::make_pair("UserDefinedDistribution", [&](auto && node){return make(node, scope);}))) // clang-format on { } From 72eda4879ebe037bb3cebcdc13a733ce7e2092c6 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:10:21 +0900 Subject: [PATCH 03/52] refactor(openscenario_interpreter): add some ifdef blocks to avoid ROS dependency --- .../reader/attribute.hpp | 25 +++++++++++++------ .../openscenario_interpreter/scope.hpp | 4 +++ .../openscenario_interpreter/src/scope.cpp | 10 ++++++-- .../src/syntax/open_scenario_category.cpp | 8 +++++- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp index 06568802cbe..1de4c2bdf42 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp @@ -15,9 +15,12 @@ #ifndef OPENSCENARIO_INTERPRETER__READER__ATTRIBUTE_HPP_ #define OPENSCENARIO_INTERPRETER__READER__ATTRIBUTE_HPP_ +#ifndef WITHOUT_ROS #include -#include #include +#endif // WITHOUT_ROS + +#include #include #include #include @@ -37,6 +40,7 @@ auto substitute(std::string attribute, Scope & scope) { auto dirname = [](auto &&, auto && scope) { return scope.dirname(); }; +#ifndef WITHOUT_ROS auto find_pkg_share = [](auto && package_name, auto &&) { return ament_index_cpp::get_package_share_directory(package_name); }; @@ -60,9 +64,11 @@ auto substitute(std::string attribute, Scope & scope) return result; } }; +#endif // WITHOUT_ROS auto var = [](auto && name, auto && scope) { - // TODO: Return the value of the launch configuration variable instead of the OpenSCENARIO parameter. + // TODO: Return the value of the launch configuration variable instead of the OpenSCENARIO + // parameter. if (const auto found = scope.ref(name); found) { return boost::lexical_cast(found); } else { @@ -75,14 +81,16 @@ auto substitute(std::string attribute, Scope & scope) std::string, std::function > substitutions{ {"dirname", dirname}, - // TODO {"env", env}, - // TODO {"eval", eval}, - // TODO {"exec-in-package", exec_in_package}, - // TODO {"find-exec", find_exec}, - // TODO {"find-pkg-prefix", find_pkg_prefix}, + // TODO {"env", env}, + // TODO {"eval", eval}, + // TODO {"exec-in-package", exec_in_package}, + // TODO {"find-exec", find_exec}, + // TODO {"find-pkg-prefix", find_pkg_prefix}, +#ifndef WITHOUT_ROS {"find-pkg-share", find_pkg_share}, {"ros2", ros2}, // NOTE: TIER IV extension (Not included in the ROS 2 Launch XML Substitution) +#endif // WITHOUT_ROS {"var", var}, }; @@ -135,7 +143,8 @@ auto readAttribute(const std::string & name, const Node & node, const Scope & sc } }; - // NOTE: https://www.asam.net/index.php?eID=dumpFile&t=f&f=4092&token=d3b6a55e911b22179e3c0895fe2caae8f5492467#_parameters + // NOTE: + // https://www.asam.net/index.php?eID=dumpFile&t=f&f=4092&token=d3b6a55e911b22179e3c0895fe2caae8f5492467#_parameters if (const auto & attribute = node.attribute(name.c_str())) { // NOTE: `substitute` is TIER IV extension (Non-OpenSCENARIO standard) diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp index 1f5a8ba8968..5897efc2d03 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp @@ -22,7 +22,9 @@ #include #include #include +#ifndef WITHOUT_ROS #include +#endif #include #include #include @@ -191,7 +193,9 @@ class Scope public: const std::string name; +#ifndef WITHOUT_ROS std::list actors; +#endif // WITHOUT_ROS // NOTE: `random_engine` is used only for sharing random number generator in Stochastic now std::mt19937 random_engine; diff --git a/openscenario/openscenario_interpreter/src/scope.cpp b/openscenario/openscenario_interpreter/src/scope.cpp index 8407b8898b8..49ec94085d9 100644 --- a/openscenario/openscenario_interpreter/src/scope.cpp +++ b/openscenario/openscenario_interpreter/src/scope.cpp @@ -18,9 +18,12 @@ #include #include #include -#include #include +#ifndef WITHOUT_ROS +#include +#endif // WITHOUT_ROS + namespace openscenario_interpreter { EnvironmentFrame::EnvironmentFrame(EnvironmentFrame & outer_frame, const std::string & name) @@ -99,8 +102,11 @@ Scope::Scope(const std::string & name, const Scope & outer) : open_scenario(outer.open_scenario), frame(std::shared_ptr(new EnvironmentFrame(*outer.frame, name))), scenario_definition(outer.scenario_definition), - name(name), + name(name) +#ifndef WITHOUT_ROS + , actors(outer.actors) +#endif // WITHOUT_ROS { } diff --git a/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp b/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp index c065a989c5f..f0967dc382d 100644 --- a/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp +++ b/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp @@ -12,10 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef WITHOUT_ROS #include +#include +#endif // WITHOUT_ROS + +#include #include #include -#include namespace openscenario_interpreter { @@ -25,8 +29,10 @@ OpenScenarioCategory::OpenScenarioCategory(const pugi::xml_node & tree, Scope & : Group( // clang-format off choice(tree, +#ifndef WITHOUT_ROS std::make_pair("Storyboard", [&](auto &&) { return make(tree, scope);}), // DIRTY HACK!!! std::make_pair("Catalog", [&](auto &&) { return make(tree, scope);}), +#endif // WITHOUT_ROS std::make_pair("ParameterValueDistribution",[&](auto &&) { return make(tree, scope);}))) // clang-format on { From 90427c599e683ced267e7758f35499c6c04cade4 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:11:41 +0900 Subject: [PATCH 04/52] feat(openscenario_preprocessor): add command executable --- .../openscenario_preprocessor/CMakeLists.txt | 5 ++ .../openscenario_preprocessor.hpp | 2 +- .../src/openscenario_preprocessor_command.cpp | 59 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 9c6d69fbcaf..a41ed90370f 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -19,7 +19,12 @@ ament_auto_add_executable(${PROJECT_NAME}_node src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_node.cpp) +ament_auto_add_executable(${PROJECT_NAME}_command + src/${PROJECT_NAME}.cpp + src/${PROJECT_NAME}_command.cpp) + target_link_libraries(${PROJECT_NAME}_node XercesC::XercesC) +target_link_libraries(${PROJECT_NAME}_command XercesC::XercesC) if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp index ba04f6d72bf..2f8a99293df 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp @@ -49,9 +49,9 @@ class Preprocessor } } -protected: void preprocessScenario(const Scenario &); +protected: std::queue preprocessed_scenarios; std::mutex preprocessed_scenarios_mutex; diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp new file mode 100644 index 00000000000..eef29e84f79 --- /dev/null +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -0,0 +1,59 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +int main(const int argc, char const * const * const argv) +{ + using namespace boost::program_options; + + options_description description("openscenario_preprocessor_command"); + + // -o -p param.json -s scenario.yaml + description.add_options()( + "output-directory,o", value()->default_value("/tmp/openscenario_preprocessor"), + "path of output directory")( + "parameters,p", value()->default_value(""), "parameters in json format")( + "scenario,s", value(), "path of scenario file")("help,H", "help"); + + std::cout << "setup description" << std::endl; + + variables_map vm; + store(parse_command_line(argc, argv, description), vm); + notify(vm); + + std::cout << "setup variables map" << std::endl; + + auto output_directory_option = boost::filesystem::path(vm["output-directory"].as()); + auto parameters_option = boost::filesystem::path(vm["parameters"].as()); + auto scenario_option = boost::filesystem::path(vm["scenario"].as()); + + std::cout << "get options" << std::endl; + + openscenario_preprocessor::Preprocessor preprocessor(output_directory_option); + + std::cout << "create preprocessor" << std::endl; + + openscenario_preprocessor::Scenario scenario; + scenario.path = scenario_option; + scenario.expect = 1; + scenario.frame_rate = 30.0; + + preprocessor.preprocessScenario(scenario); + + std::cout << "preprocess scenario" << std::endl; + + return 0; +} From b1200cac26abfcc07c8c2284bec147469d477faa Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:12:20 +0900 Subject: [PATCH 05/52] feat(openscenario_interpreter): add library without ROS dependency (WIP) --- .../openscenario_interpreter/CMakeLists.txt | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/openscenario/openscenario_interpreter/CMakeLists.txt b/openscenario/openscenario_interpreter/CMakeLists.txt index 7e9f17620a6..dbfe2b668c6 100644 --- a/openscenario/openscenario_interpreter/CMakeLists.txt +++ b/openscenario/openscenario_interpreter/CMakeLists.txt @@ -54,6 +54,53 @@ install( FILES ${CMAKE_CURRENT_SOURCE_DIR}/features DESTINATION lib/${PROJECT_NAME}) +# ------------------------------------------------------------------------------ +# parameter value distribution without ROS +# ------------------------------------------------------------------------------ +add_library(parameter_value_distribution_without_ros SHARED + src/syntax/file_header.cpp + src/syntax/license.cpp + src/syntax/properties.cpp + src/syntax/property.cpp + src/syntax/file.cpp + src/syntax/open_scenario_category.cpp + src/syntax/parameter_value_distribution_definition.cpp + src/syntax/parameter_value_distribution.cpp + src/syntax/distribution_definition.cpp + src/syntax/deterministic.cpp + src/syntax/deterministic_parameter_distribution.cpp + src/syntax/deterministic_multi_parameter_distribution.cpp + src/syntax/deterministic_multi_parameter_distribution_type.cpp + src/syntax/value_set_distribution.cpp + src/syntax/deterministic_single_parameter_distribution.cpp + src/syntax/deterministic_single_parameter_distribution_type.cpp + src/syntax/distribution_range.cpp + src/syntax/double.cpp + src/syntax/distribution_set.cpp + src/syntax/distribution_set_element.cpp + src/syntax/stochastic.cpp + src/syntax/unsigned_integer.cpp + src/syntax/stochastic_distribution.cpp + src/syntax/stochastic_distribution_type.cpp + src/syntax/histogram.cpp + src/syntax/histogram_bin.cpp + src/syntax/normal_distribution.cpp + src/syntax/poisson_distribution.cpp + src/syntax/probability_distribution_set.cpp + src/syntax/probability_distribution_set_element.cpp + src/syntax/uniform_distribution.cpp + + src/object.cpp + src/parameter_distribution.cpp + src/scope.cpp) +add_definitions("-DBOOST_ALLOW_DEPRECATED_HEADERS") +target_compile_definitions(parameter_value_distribution_without_ros PUBLIC -DWITHOUT_ROS) +target_include_directories(parameter_value_distribution_without_ros PUBLIC include) +target_include_directories(parameter_value_distribution_without_ros PUBLIC ${scenario_simulator_exception_INCLUDE_DIRS}) +target_link_libraries(parameter_value_distribution_without_ros + Boost::filesystem + pugixml) + # ------------------------------------------------------------------------------ # test # ------------------------------------------------------------------------------ From fa4cca005dec1b834274746f306e1d17f5c425ae Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:13:12 +0900 Subject: [PATCH 06/52] chore: apply linter --- .../include/openscenario_interpreter/syntax/integer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp index afaf8f073d0..aa50692211d 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/syntax/integer.hpp @@ -15,8 +15,8 @@ #ifndef OPENSCENARIO_INTERPRETER__SYNTAX__INTEGER_HPP_ #define OPENSCENARIO_INTERPRETER__SYNTAX__INTEGER_HPP_ -#include #include +#include #include namespace openscenario_interpreter From a0304e3a1d07f4dd09df296865787750aa421b94 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:27:14 +0900 Subject: [PATCH 07/52] chore: apply linter --- openscenario/openscenario_validator/package.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/openscenario/openscenario_validator/package.xml b/openscenario/openscenario_validator/package.xml index be2bfdb3da6..5e4a4ccf1ac 100644 --- a/openscenario/openscenario_validator/package.xml +++ b/openscenario/openscenario_validator/package.xml @@ -12,4 +12,3 @@ ament_cmake - From c185e38da27d2c59eda534563b6ce46165b430ee Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:44:04 +0900 Subject: [PATCH 08/52] chore: fix cspell error --- openscenario/openscenario_interpreter/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/openscenario/openscenario_interpreter/CMakeLists.txt b/openscenario/openscenario_interpreter/CMakeLists.txt index dbfe2b668c6..3531eed569c 100644 --- a/openscenario/openscenario_interpreter/CMakeLists.txt +++ b/openscenario/openscenario_interpreter/CMakeLists.txt @@ -94,6 +94,7 @@ add_library(parameter_value_distribution_without_ros SHARED src/parameter_distribution.cpp src/scope.cpp) add_definitions("-DBOOST_ALLOW_DEPRECATED_HEADERS") +# cspell: ignore DWITHOUT target_compile_definitions(parameter_value_distribution_without_ros PUBLIC -DWITHOUT_ROS) target_include_directories(parameter_value_distribution_without_ros PUBLIC include) target_include_directories(parameter_value_distribution_without_ros PUBLIC ${scenario_simulator_exception_INCLUDE_DIRS}) From 4d7c386179722535c3ad8722bf5bc44bbde30496 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 10:56:25 +0900 Subject: [PATCH 09/52] chore: modify cspell setting --- .github/workflows/custom_spell.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom_spell.json b/.github/workflows/custom_spell.json index 8c47782c42b..c85420bd0f1 100644 --- a/.github/workflows/custom_spell.json +++ b/.github/workflows/custom_spell.json @@ -3,7 +3,7 @@ "*.drawio", "*/**/CHANGELOG.rst", "external/lanelet2_matching/**", - "openscenario/openscenario_utility/openscenario_utility/resources/*.xsd" + "*.xsd" ], "words": [ "ARGN", From 7ea1801a12d89ed66cc83523961d00f51bff39f3 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 16:03:42 +0900 Subject: [PATCH 10/52] refactor(openscenario_interpreter): change macro name --- .../openscenario_interpreter/reader/attribute.hpp | 12 ++++++------ .../include/openscenario_interpreter/scope.hpp | 10 ++++++---- openscenario/openscenario_interpreter/src/scope.cpp | 8 ++++---- .../src/syntax/open_scenario.cpp | 5 +++++ .../src/syntax/open_scenario_category.cpp | 8 ++++---- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp index 1de4c2bdf42..36bc11c5c9e 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/reader/attribute.hpp @@ -15,10 +15,10 @@ #ifndef OPENSCENARIO_INTERPRETER__READER__ATTRIBUTE_HPP_ #define OPENSCENARIO_INTERPRETER__READER__ATTRIBUTE_HPP_ -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY #include #include -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY #include #include @@ -40,7 +40,7 @@ auto substitute(std::string attribute, Scope & scope) { auto dirname = [](auto &&, auto && scope) { return scope.dirname(); }; -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY auto find_pkg_share = [](auto && package_name, auto &&) { return ament_index_cpp::get_package_share_directory(package_name); }; @@ -64,7 +64,7 @@ auto substitute(std::string attribute, Scope & scope) return result; } }; -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY auto var = [](auto && name, auto && scope) { // TODO: Return the value of the launch configuration variable instead of the OpenSCENARIO @@ -86,11 +86,11 @@ auto substitute(std::string attribute, Scope & scope) // TODO {"exec-in-package", exec_in_package}, // TODO {"find-exec", find_exec}, // TODO {"find-pkg-prefix", find_pkg_prefix}, -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY {"find-pkg-share", find_pkg_share}, {"ros2", ros2}, // NOTE: TIER IV extension (Not included in the ROS 2 Launch XML Substitution) -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY {"var", var}, }; diff --git a/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp b/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp index 5897efc2d03..dae4ac41b10 100644 --- a/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp +++ b/openscenario/openscenario_interpreter/include/openscenario_interpreter/scope.hpp @@ -22,9 +22,11 @@ #include #include #include -#ifndef WITHOUT_ROS + +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY #include -#endif +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY + #include #include #include @@ -193,9 +195,9 @@ class Scope public: const std::string name; -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY std::list actors; -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY // NOTE: `random_engine` is used only for sharing random number generator in Stochastic now std::mt19937 random_engine; diff --git a/openscenario/openscenario_interpreter/src/scope.cpp b/openscenario/openscenario_interpreter/src/scope.cpp index 49ec94085d9..c5d5ff8dc16 100644 --- a/openscenario/openscenario_interpreter/src/scope.cpp +++ b/openscenario/openscenario_interpreter/src/scope.cpp @@ -20,9 +20,9 @@ #include #include -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY #include -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY namespace openscenario_interpreter { @@ -103,10 +103,10 @@ Scope::Scope(const std::string & name, const Scope & outer) frame(std::shared_ptr(new EnvironmentFrame(*outer.frame, name))), scenario_definition(outer.scenario_definition), name(name) -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY , actors(outer.actors) -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY { } diff --git a/openscenario/openscenario_interpreter/src/syntax/open_scenario.cpp b/openscenario/openscenario_interpreter/src/syntax/open_scenario.cpp index 638e9d00148..fee55f6908a 100644 --- a/openscenario/openscenario_interpreter/src/syntax/open_scenario.cpp +++ b/openscenario/openscenario_interpreter/src/syntax/open_scenario.cpp @@ -15,7 +15,10 @@ #include #include #include + +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY #include +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY namespace openscenario_interpreter { @@ -48,6 +51,7 @@ auto operator<<(nlohmann::json & json, const OpenScenario & datum) -> nlohmann:: { json["version"] = "1.0"; +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY json["frame"] = datum.frame; // clang-format off @@ -61,6 +65,7 @@ auto operator<<(nlohmann::json & json, const OpenScenario & datum) -> nlohmann:: if (datum.category.is()) { json["OpenSCENARIO"] << datum.category.as(); } +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY return json; } diff --git a/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp b/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp index f0967dc382d..e47334b8854 100644 --- a/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp +++ b/openscenario/openscenario_interpreter/src/syntax/open_scenario_category.cpp @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY #include #include -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY #include #include @@ -29,10 +29,10 @@ OpenScenarioCategory::OpenScenarioCategory(const pugi::xml_node & tree, Scope & : Group( // clang-format off choice(tree, -#ifndef WITHOUT_ROS +#ifndef PARAMETER_VALUE_DISTRIBUTION_ONLY std::make_pair("Storyboard", [&](auto &&) { return make(tree, scope);}), // DIRTY HACK!!! std::make_pair("Catalog", [&](auto &&) { return make(tree, scope);}), -#endif // WITHOUT_ROS +#endif // PARAMETER_VALUE_DISTRIBUTION_ONLY std::make_pair("ParameterValueDistribution",[&](auto &&) { return make(tree, scope);}))) // clang-format on { From 30c5251019e4cfcc61e4ab13a5db140ad8cef2d5 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 26 Apr 2023 16:04:23 +0900 Subject: [PATCH 11/52] fix: add necessary build settings --- .../openscenario_interpreter/CMakeLists.txt | 25 +++++++++++++++--- .../openscenario_preprocessor/CMakeLists.txt | 26 ++++++++++++++++--- .../openscenario_preprocessor/package.xml | 1 + 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/openscenario/openscenario_interpreter/CMakeLists.txt b/openscenario/openscenario_interpreter/CMakeLists.txt index 3531eed569c..df29d823e22 100644 --- a/openscenario/openscenario_interpreter/CMakeLists.txt +++ b/openscenario/openscenario_interpreter/CMakeLists.txt @@ -11,6 +11,7 @@ if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") endif() find_package(ament_cmake_auto REQUIRED) +find_package(Boost REQUIRED COMPONENTS filesystem) ament_auto_find_build_dependencies() @@ -57,7 +58,8 @@ install( # ------------------------------------------------------------------------------ # parameter value distribution without ROS # ------------------------------------------------------------------------------ -add_library(parameter_value_distribution_without_ros SHARED +add_library(parameter_value_distribution_without_ros STATIC + src/syntax/open_scenario.cpp src/syntax/file_header.cpp src/syntax/license.cpp src/syntax/properties.cpp @@ -75,11 +77,10 @@ add_library(parameter_value_distribution_without_ros SHARED src/syntax/deterministic_single_parameter_distribution.cpp src/syntax/deterministic_single_parameter_distribution_type.cpp src/syntax/distribution_range.cpp - src/syntax/double.cpp + src/syntax/range.cpp src/syntax/distribution_set.cpp src/syntax/distribution_set_element.cpp src/syntax/stochastic.cpp - src/syntax/unsigned_integer.cpp src/syntax/stochastic_distribution.cpp src/syntax/stochastic_distribution_type.cpp src/syntax/histogram.cpp @@ -90,18 +91,34 @@ add_library(parameter_value_distribution_without_ros SHARED src/syntax/probability_distribution_set_element.cpp src/syntax/uniform_distribution.cpp + src/syntax/boolean.cpp + src/syntax/double.cpp + src/syntax/integer.cpp + src/syntax/unsigned_integer.cpp + src/syntax/unsigned_short.cpp + + src/syntax/parameter_value_set.cpp + + src/utility/demangle.cpp + + src/evaluate.cpp src/object.cpp src/parameter_distribution.cpp src/scope.cpp) + add_definitions("-DBOOST_ALLOW_DEPRECATED_HEADERS") # cspell: ignore DWITHOUT -target_compile_definitions(parameter_value_distribution_without_ros PUBLIC -DWITHOUT_ROS) +target_compile_definitions(parameter_value_distribution_without_ros PUBLIC -DPARAMETER_VALUE_DISTRIBUTION_ONLY) target_include_directories(parameter_value_distribution_without_ros PUBLIC include) target_include_directories(parameter_value_distribution_without_ros PUBLIC ${scenario_simulator_exception_INCLUDE_DIRS}) target_link_libraries(parameter_value_distribution_without_ros Boost::filesystem pugixml) +install( + TARGETS parameter_value_distribution_without_ros + LIBRARY DESTINATION lib/${PROJECT_NAME}) + # ------------------------------------------------------------------------------ # test # ------------------------------------------------------------------------------ diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index a41ed90370f..8189b434b40 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.16.3) # Ubuntu 20.04 default CMake version project(openscenario_preprocessor) if(NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD 17) endif() if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") @@ -12,6 +12,8 @@ endif() find_package(ament_cmake_auto REQUIRED) find_package(XercesC REQUIRED) +find_package(Boost REQUIRED COMPONENTS filesystem program_options) +find_package(pugixml REQUIRED) ament_auto_find_build_dependencies() @@ -19,12 +21,30 @@ ament_auto_add_executable(${PROJECT_NAME}_node src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_node.cpp) -ament_auto_add_executable(${PROJECT_NAME}_command +target_link_libraries(${PROJECT_NAME}_node XercesC::XercesC) + +find_library(parameter_value_distribution_LIBRARYIES parameter_value_distribution_without_ros) + +add_executable(${PROJECT_NAME}_command src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_command.cpp) -target_link_libraries(${PROJECT_NAME}_node XercesC::XercesC) +target_compile_definitions(${PROJECT_NAME}_command PUBLIC -DPARAMETER_VALUE_DISTRIBUTION_ONLY) + +target_include_directories(${PROJECT_NAME}_command PUBLIC include) +target_include_directories(${PROJECT_NAME}_command PUBLIC ${XercesC_INCLUDE_DIRS}) +target_include_directories(${PROJECT_NAME}_command PUBLIC ${openscenario_interpreter_INCLUDE_DIRS}) +target_include_directories(${PROJECT_NAME}_command PUBLIC ${openscenario_validator_INCLUDE_DIRS}) + +target_link_libraries(${PROJECT_NAME}_command Boost::filesystem Boost::program_options) target_link_libraries(${PROJECT_NAME}_command XercesC::XercesC) +target_link_libraries(${PROJECT_NAME}_command pugixml) +target_link_libraries(${PROJECT_NAME}_command ${parameter_value_distribution_LIBRARYIES}) +target_link_libraries(${PROJECT_NAME}_command ${openscenario_validator_LIBRARIES}) + +install(TARGETS ${PROJECT_NAME}_node ${PROJECT_NAME}_command + RUNTIME DESTINATION lib/${PROJECT_NAME} + ) if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) diff --git a/openscenario/openscenario_preprocessor/package.xml b/openscenario/openscenario_preprocessor/package.xml index 590b628ee5b..d001e756f2f 100644 --- a/openscenario/openscenario_preprocessor/package.xml +++ b/openscenario/openscenario_preprocessor/package.xml @@ -14,6 +14,7 @@ openscenario_validator rclcpp xerces + pugixml-dev ament_cmake_clang_format ament_cmake_copyright From 7d68ba8ddfefece6e9f3bf29ff6df8541f4848e4 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Thu, 27 Apr 2023 14:45:20 +0900 Subject: [PATCH 12/52] fix(openscenario_preprocessor): disable reference scenario loading by OpenScenario class --- .../src/openscenario_preprocessor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index 2e21b872c90..4bcb30bfc4f 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -37,14 +37,15 @@ void Preprocessor::preprocessScenario(const Scenario & scenario) if (boost::filesystem::exists(scenario_file_path)) { validate(scenario_file_path); - OpenScenario scenario_file{scenario_file_path}; + pugi::xml_document scenario_file_doc; + scenario_file_doc.load_file(scenario_file_path.c_str()); auto p = parameter_value_distribution.derive(); for (const auto & parameter_list : p | boost::adaptors::indexed()) { pugi::xml_document derived_script; - derived_script.reset(scenario_file.script); // deep copy + derived_script.reset(scenario_file_doc); // deep copy auto parameter_declarations = derived_script.document_element() From c20acc969e37e562c6e4bd0f302ebcaf818c2ad8 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 28 Apr 2023 19:35:54 +0900 Subject: [PATCH 13/52] chore(openscenario_preprocessor): add template scenarios --- .../template_distributions.hpp | 27 ++ .../openscenario_preprocessor/tojson.hpp | 383 ++++++++++++++++++ .../src/template_distribution_range.cpp | 33 ++ .../src/template_value_set_distribution.cpp | 34 ++ 4 files changed, 477 insertions(+) create mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp create mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp create mode 100644 openscenario/openscenario_preprocessor/src/template_distribution_range.cpp create mode 100644 openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp new file mode 100644 index 00000000000..7baaf4d9c8e --- /dev/null +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp @@ -0,0 +1,27 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPENSCENARIO_PREPROCESSOR_TEMPLATE_DISTRIBUTIONS_HPP +#define OPENSCENARIO_PREPROCESSOR_TEMPLATE_DISTRIBUTIONS_HPP + +#include + +namespace openscenario_preprocessor +{ +extern const std::string_view template_value_set_distribution; + +extern const std::string_view template_distribution_range; +} // namespace openscenario_preprocessor + +#endif //OPENSCENARIO_PREPROCESSOR_TEMPLATE_DISTRIBUTIONS_HPP diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp new file mode 100644 index 00000000000..435c56dcb4d --- /dev/null +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp @@ -0,0 +1,383 @@ +// MIT License +// +// Copyright (c) 2021 Mirco +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef OPENSCENARIO_PREPROCESSOR_TOJSON_HPP +#define OPENSCENARIO_PREPROCESSOR_TOJSON_HPP + +#include + +#include +#include +#include +#include + +/* Adding declarations to make it compatible with gcc 4.7 and greater */ +#if __GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ > 40700 +namespace rapidxml +{ +namespace internal +{ +template +inline OutIt print_children(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_attributes(OutIt out, const xml_node * node, int flags); +template +inline OutIt print_data_node(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_cdata_node(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_element_node(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_declaration_node(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_comment_node(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_doctype_node(OutIt out, const xml_node * node, int flags, int indent); +template +inline OutIt print_pi_node(OutIt out, const xml_node * node, int flags, int indent); +} // namespace internal +} // namespace rapidxml +#include +#endif + +#if __has_cpp_attribute(nodiscard) +#define TOJSON_NODISCARD [[nodiscard]] +#else +#define TOJSON_NODISCARD +#endif + +#include +using json = nlohmann::ordered_json; + +namespace tojson +{ +namespace detail +{ + +/// \todo refactor and pass nlohmann::json down by reference instead of returning it +inline json xml2json(const rapidxml::xml_node<> * root) +{ + json j{}; + std::map occurrence{}; + + for (auto * node = root->first_node(); node; node = node->next_sibling()) { + auto key = node->name(); + occurrence[key] += 1; + + // if seen current node, convert it into a list + if (occurrence[key] == 2) j[key] = {j[key]}; + + // if the node is a sequence + if (occurrence[key] > 1) { + json n{}; + if (node->first_node()) + n[key] = xml2json(node); + else + n["@text"] = node->value(); + // toyaml through the attributes + for (auto * attr = node->first_attribute(); attr; attr = attr->next_attribute()) + n[key][attr->name()] = attr->value(); + j[key].emplace_back(n[key]); + } else { + if (node->first_node()) + j[key] = xml2json(node); + else + j["@text"] = node->value(); + // toyaml through the attributes + for (auto * attr = node->first_attribute(); attr; attr = attr->next_attribute()) + j[key][attr->name()] = attr->value(); + } + } + + return j; +} + +/// \todo refactor and pass nlohmann::json down by reference instead of returning it +inline json pugixml2json(const pugi::xml_node & root) +{ + json j{}; + std::map occurrence{}; + + for (auto & child : root.children()) { + auto key = child.name(); + occurrence[key] += 1; + + // if seen current node, convert it into a list + if (occurrence[key] == 2) j[key] = json::array({j[key]}); + + // if the node is a sequence + if (occurrence[key] > 1) { + json n{}; + if (child == child.parent().first_child()) + n[key].emplace_back(pugixml2json(child)); + else +// n[key].emplace_back(child.value()); + + n["@text"] = child.value(); + // toyaml through the attributes + for (auto & attr : child.attributes()) { + n[key][attr.name()] = attr.value(); + } + + j[key].emplace_back(n[key]); + } else { + if (child == child.parent().first_child()) + j[key] = pugixml2json(child); + else +// j[key].emplace_back(child.value()); + j["@text"] = child.value(); + // toyaml through the attributes + for (auto & attr : child.attributes()) { + j[key][attr.name()] = attr.value(); + } + } + } + + return j; +} + +inline json parse_scalar(const YAML::Node & node) +{ + int i; + double d; + bool b; + std::string s; + + if (YAML::convert::decode(node, i)) return i; + if (YAML::convert::decode(node, d)) return d; + if (YAML::convert::decode(node, b)) return b; + if (YAML::convert::decode(node, s)) return s; + + return nullptr; +} + +/// \todo refactor and pass nlohmann::json down by reference instead of returning it +inline json yaml2json(const YAML::Node & root) +{ + json j{}; + + switch (root.Type()) { + case YAML::NodeType::Null: + break; + case YAML::NodeType::Scalar: + return parse_scalar(root); + case YAML::NodeType::Sequence: + for (auto && node : root) j.emplace_back(yaml2json(node)); + break; + case YAML::NodeType::Map: + for (auto && it : root) j[it.first.as()] = yaml2json(it.second); + break; + default: + break; + } + return j; +} + +/// \todo handle @text entries better +inline void toyaml(const json & j, YAML::Emitter & e) +{ + for (auto it = j.begin(); it != j.end(); ++it) { + if (it->is_object()) { + e << YAML::Key << it.key() << YAML::Value << YAML::BeginMap; + toyaml(*it, e); + e << YAML::EndMap; + } else if (it->is_array()) { + e << YAML::Key << it.key() << YAML::Value << YAML::BeginSeq; + toyaml(it.value(), e); + e << YAML::EndSeq; + } else { + if (it.key() == "@text") { + e << YAML::Value << it.value().get(); + } else { + e << YAML::Key << it.key() << YAML::Value << it.value().get(); + } + } + } +} + +// Forward declaration required here for circular dipedency. +inline void toxml(const json & j, rapidxml::xml_document<> & doc, rapidxml::xml_node<> * parent); + +inline std::string repr(const json & j) +{ + if (j.is_number()) return std::to_string(j.get()); + if (j.is_boolean()) return j.get() ? "true" : "false"; + if (j.is_number_float()) return std::to_string(j.get()); + if (j.is_string()) return j.get(); + std::runtime_error("invalid type"); + return ""; +} + +/// \todo handle @text entries better +inline void toxml( + const json & j, rapidxml::xml_document<> & doc, rapidxml::xml_node<> * parent, + const std::string & key) +{ + // std::cout << "toxml for array : " << key << std::endl; + // Not the prettiest of designs, but it works fine. + for (auto it = j.begin(); it != j.end(); ++it) { + if (it->is_object()) { + // std::cout << "object in array : " << *it << std::endl; + // std::cout << it.key() << std::endl; + // auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(it.key().data())); + auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(key.data())); + // auto * node = doc.allocate_node(rapidxml::node_element); + detail::toxml(*it, doc, node); + parent->append_node(node); + } else if (it->is_array()) { + // std::cout << "array in array" << std::endl; + detail::toxml(*it, doc, parent, key); + } else { + // std::cout << "else in array : " << std::endl; + // std::cout << it.key() << std::endl; + auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(key.data())); + node->value(doc.allocate_string(repr(it.value()).data())); + parent->append_node(node); + } + } +} + +/// \todo handle @text entries better +inline void toxml(const json & j, rapidxml::xml_document<> & doc, rapidxml::xml_node<> * parent) +{ + // std::cout << "detail::toxml 1" << std::endl; + for (auto it = j.begin(); it != j.end(); ++it) { + if (it->is_object()) { + auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(it.key().data())); + detail::toxml(*it, doc, node); + parent->append_node(node); + } else if (it->is_array()) { + detail::toxml(*it, doc, parent, it.key()); + } else { + // std::cout << "detail::toxml else : "<< it.key().data() << std::endl; + if (it->is_null()) { + detail::toxml(*it, doc, parent, it.key()); + + } else { + auto * node = doc.allocate_attribute(doc.allocate_string(it.key().data())); + node->value(doc.allocate_string(repr(it.value()).data())); + parent->append_attribute(node); + } + } + } +} + +} // namespace detail + +/// \brief Convert XML string to JSON. +TOJSON_NODISCARD inline json xml2json(const std::string & str) +{ + std::string replacement = "\\/"; + boost::regex re("(?)"); + std::string escaped_str = boost::regex_replace(str, re, replacement); + + json j{}; + rapidxml::xml_document<> doc{}; + doc.parse<0>(const_cast(escaped_str.data())); + + auto * root = doc.first_node(); + if (root) j[root->name()] = detail::xml2json(root); + + return j; +} + +TOJSON_NODISCARD inline json pugixml2json(pugi::xml_node & root) +{ + json j{}; + j[root.name()] = detail::pugixml2json(root); + + return j; +} + +/// \brief Convert YAML string to JSON. +TOJSON_NODISCARD inline json yaml2json(const std::string & str) +{ + YAML::Node root = YAML::Load(str); + return detail::yaml2json(root); +} + +/// \brief Load a YAML file to JSON. +TOJSON_NODISCARD inline json loadyaml(const std::string & filepath) +{ + YAML::Node root = YAML::LoadFile(filepath); + return detail::yaml2json(root); +} + +/// \brief Load XML file to JSON. +TOJSON_NODISCARD inline json loadxml(const std::string & filepath) +{ + std::ifstream file{filepath.data()}; + std::string str{std::istream_iterator(file), std::istream_iterator()}; + // std::cout << "loaded : " << str << std::endl; + return xml2json(str); +} + +namespace emitters +{ + +/// \brief Generate string representation of json as an YAML document. +TOJSON_NODISCARD inline std::string toyaml(const json & j) +{ + YAML::Emitter e; + e << YAML::BeginDoc; + if (j.is_object()) { + e << YAML::BeginMap; + detail::toyaml(j, e); + e << YAML::EndMap; + } else if (j.is_array()) { + e << YAML::BeginSeq; + detail::toyaml(j, e); + e << YAML::EndSeq; + } + e << YAML::EndDoc; + return e.c_str(); +} + +/// \brief Generate string representation of json as an XML document. +/// \param j Json object to convert, must have a single root +/// \throws std::runtime_error if the object have more than one root +TOJSON_NODISCARD inline std::string toxml(const json & j) +{ + rapidxml::xml_document<> doc; + auto * decl = doc.allocate_node(rapidxml::node_declaration); + decl->append_attribute(doc.allocate_attribute("version", "1.0")); + decl->append_attribute(doc.allocate_attribute("encoding", "utf-8")); + doc.append_node(decl); + + if (j.is_object() && j.size() == 1) { + auto * root = + doc.allocate_node(rapidxml::node_element, doc.allocate_string(j.begin().key().data())); + detail::toxml(j.begin().value(), doc, root); + doc.append_node(root); + } else { + throw std::runtime_error("json must have a single root node"); + } + + std::string xml_as_string; + rapidxml::print(std::back_inserter(xml_as_string), doc); + return xml_as_string; +} + +} // namespace emitters +} // namespace tojson + +#endif //OPENSCENARIO_PREPROCESSOR_TOJSON_HPP diff --git a/openscenario/openscenario_preprocessor/src/template_distribution_range.cpp b/openscenario/openscenario_preprocessor/src/template_distribution_range.cpp new file mode 100644 index 00000000000..10e7d8a3f8f --- /dev/null +++ b/openscenario/openscenario_preprocessor/src/template_distribution_range.cpp @@ -0,0 +1,33 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace openscenario_preprocessor +{ +const std::string_view template_distribution_trange = R"###( + + + + + + + + + + + + +)###"; +} // namespace openscenario_preprocessor diff --git a/openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp b/openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp new file mode 100644 index 00000000000..13fb9156c02 --- /dev/null +++ b/openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp @@ -0,0 +1,34 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace openscenario_preprocessor +{ +const std::string_view template_value_set_distribution = R"###( + + + + + + + + + + + + + +)###"; +} // namespace openscenario_preprocessor From dc1a8f06721f63fd296331333e92be5761ad2cea Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 28 Apr 2023 19:36:12 +0900 Subject: [PATCH 14/52] chore(openscenario_preprocessor): add dependencies --- openscenario/openscenario_preprocessor/CMakeLists.txt | 10 ++++++++-- openscenario/openscenario_preprocessor/package.xml | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 8189b434b40..22f05c9ea07 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -12,7 +12,7 @@ endif() find_package(ament_cmake_auto REQUIRED) find_package(XercesC REQUIRED) -find_package(Boost REQUIRED COMPONENTS filesystem program_options) +find_package(Boost REQUIRED COMPONENTS filesystem program_options regex) find_package(pugixml REQUIRED) ament_auto_find_build_dependencies() @@ -22,10 +22,12 @@ ament_auto_add_executable(${PROJECT_NAME}_node src/${PROJECT_NAME}_node.cpp) target_link_libraries(${PROJECT_NAME}_node XercesC::XercesC) +target_link_libraries(${PROJECT_NAME}_node yaml-cpp) find_library(parameter_value_distribution_LIBRARYIES parameter_value_distribution_without_ros) add_executable(${PROJECT_NAME}_command + src/t4v2.cpp src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_command.cpp) @@ -33,12 +35,16 @@ target_compile_definitions(${PROJECT_NAME}_command PUBLIC -DPARAMETER_VALUE_DIST target_include_directories(${PROJECT_NAME}_command PUBLIC include) target_include_directories(${PROJECT_NAME}_command PUBLIC ${XercesC_INCLUDE_DIRS}) +target_include_directories(${PROJECT_NAME}_command PUBLIC ${YAML_CPP_INCLUDE_DIRS}) +target_include_directories(${PROJECT_NAME}_command PUBLIC ${YAML_CPP_INCLUDE_DIRS}) target_include_directories(${PROJECT_NAME}_command PUBLIC ${openscenario_interpreter_INCLUDE_DIRS}) target_include_directories(${PROJECT_NAME}_command PUBLIC ${openscenario_validator_INCLUDE_DIRS}) -target_link_libraries(${PROJECT_NAME}_command Boost::filesystem Boost::program_options) +target_link_libraries(${PROJECT_NAME}_command Boost::filesystem Boost::program_options Boost::regex) target_link_libraries(${PROJECT_NAME}_command XercesC::XercesC) target_link_libraries(${PROJECT_NAME}_command pugixml) +target_link_libraries(${PROJECT_NAME}_command yaml-cpp) + target_link_libraries(${PROJECT_NAME}_command ${parameter_value_distribution_LIBRARYIES}) target_link_libraries(${PROJECT_NAME}_command ${openscenario_validator_LIBRARIES}) diff --git a/openscenario/openscenario_preprocessor/package.xml b/openscenario/openscenario_preprocessor/package.xml index d001e756f2f..dc58526b414 100644 --- a/openscenario/openscenario_preprocessor/package.xml +++ b/openscenario/openscenario_preprocessor/package.xml @@ -15,6 +15,9 @@ rclcpp xerces pugixml-dev + yaml-cpp + librapidxml-dev + nlohmann-json-dev ament_cmake_clang_format ament_cmake_copyright From 730237649fc267d6975dfab197565deb29b39611 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 28 Apr 2023 19:36:28 +0900 Subject: [PATCH 15/52] feat(openscenario_preprocessor): add t4v2 class --- .../openscenario_preprocessor/t4v2.hpp | 41 ++++ .../openscenario_preprocessor/src/t4v2.cpp | 214 ++++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp create mode 100644 openscenario/openscenario_preprocessor/src/t4v2.cpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp new file mode 100644 index 00000000000..289d76f8cf5 --- /dev/null +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp @@ -0,0 +1,41 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPENSCENARIO_PREPROCESSOR_T4V2_HPP +#define OPENSCENARIO_PREPROCESSOR_T4V2_HPP + +#include +#include +#include +#include + +namespace openscenario_preprocessor +{ +class T4V2 +{ +public: + auto deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vector; + + auto generateParameterValueDistributionFromScenarioModifiers(std::string scenario_modifiers_str) + -> pugi::xml_document; + + auto deriveScenarioWithScenarioModifiers( + const pugi::xml_document & base_scenario_doc, + const openscenario_interpreter::ParameterDistribution & scenario_modifier_distribution) + -> std::vector; + + std::pair splitScenarioModifiers(std::string scenario); +}; +} // namespace openscenario_preprocessor +#endif //OPENSCENARIO_PREPROCESSOR_T4V2_HPP diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp new file mode 100644 index 00000000000..1dc7d08263e --- /dev/null +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -0,0 +1,214 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace openscenario_preprocessor +{ +const std::string_view scenario_modifiers_distribution_base = R"###( + + + + + + + +)###"; + +auto T4V2::deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vector +{ + std::fstream file{path}; + std::stringstream scenario; + scenario << file.rdbuf(); + file.close(); + + auto split = splitScenarioModifiers(scenario.str()); + + std::ofstream base_scenario_ofs("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); + base_scenario_ofs << split.second; + base_scenario_ofs.close(); + + auto openscenario_json = + tojson::loadyaml("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); + auto openscenario_xml = tojson::emitters::toxml(openscenario_json); + pugi::xml_document openscenario_doc; + openscenario_doc.load_string(openscenario_xml.c_str()); + + std::vector derived_scenarios; + if (not split.first.empty()) { + auto parameter_value_distribution = + generateParameterValueDistributionFromScenarioModifiers(split.first); + boost::filesystem::path parameter_value_distribution_path = + "/tmp/openscenario_preprocessor/scenario_modifiers_distribution.xosc"; + parameter_value_distribution.save_file(parameter_value_distribution_path.string().c_str()); + + Deriver derive; + auto distribution = derive(parameter_value_distribution_path, false); + + if (not distribution.empty()) { + auto derived_scenario_docs = + deriveScenarioWithScenarioModifiers(openscenario_doc, distribution); + for (auto & derived_scenario_doc : derived_scenario_docs) { + std::stringstream derived_scenario_stream; + derived_scenario_doc.save(derived_scenario_stream); + derived_scenarios.push_back(derived_scenario_stream.str()); + } + } else { + // no scenario modifiers + std::stringstream scenario_stream; + openscenario_doc.save(scenario_stream); + derived_scenarios.push_back(scenario_stream.str()); + } + } else { + // no scenario modifiers + std::stringstream scenario_stream; + openscenario_doc.save(scenario_stream); + derived_scenarios.push_back(scenario_stream.str()); + } + return derived_scenarios; +} + +std::pair T4V2::splitScenarioModifiers(std::string scenario) +{ + std::regex re("(.*\n|^)(OpenSCENARIO.*)"); + std::smatch match; + + if (std::regex_search(scenario, match, re)) { + return std::make_pair( + scenario.substr(0, match.position(2)), scenario.substr(match.position(2))); + } else { + throw std::runtime_error( + "No OpenSCENARIO element found in TIER IV 2.0 Format Scenario. Please check your " + "scenario."); + } +} + +auto T4V2::generateParameterValueDistributionFromScenarioModifiers( + std::string scenario_modifiers_str) -> pugi::xml_document +{ + pugi::xml_document doc; + doc.load_string(scenario_modifiers_distribution_base.data()); + auto deterministic = + doc.select_node(pugi::xpath_query{"OpenSCENARIO/ParameterValueDistribution/Deterministic"}) + .node(); + + auto scenario_modifiers = + YAML::Load(scenario_modifiers_str)["ScenarioModifiers"]["ScenarioModifier"]; + for (const auto & scenario_modifier : scenario_modifiers) { + auto distribution = deterministic.append_child("DeterministicSingleParameterDistribution"); + + struct + { + bool list_enabled = false; + std::string name; + double start; + double step; + double stop; + std::vector list; + } data; + + for (auto modifier_element : scenario_modifier) { + auto key = modifier_element.first.as(); + + if (key == "name") { + data.name = modifier_element.second.as(); + } else if (key == "list") { + data.list_enabled = true; + for (const auto & value : modifier_element.second.as>()) { + data.list.push_back(value); + } + } else if (key == "start") { + data.start = modifier_element.second.as(); + } else if (key == "step") { + data.step = modifier_element.second.as(); + } else if (key == "stop") { + data.stop = modifier_element.second.as(); + } else { + std::cout << "unknown key: " << key << std::endl; + } + } + + distribution.append_attribute("parameterName") = data.name.c_str(); + + if (data.list_enabled) { + // add distribution set + auto distribution_set = distribution.append_child("DistributionSet"); + for (const auto & value : data.list) { + auto distribution_set_element = distribution_set.append_child("Element"); + distribution_set_element.append_attribute("value") = value.c_str(); + } + } else { + // add distribution range + auto range = distribution.append_child("DistributionRange"); + range.append_attribute("stepWidth") = data.step; + auto range_range = range.append_child("Range"); + range_range.append_attribute("upperLimit") = data.stop; + range_range.append_attribute("lowerLimit") = data.start; + } + } + return doc; +} + +auto T4V2::deriveScenarioWithScenarioModifiers( + const pugi::xml_document & base_scenario_doc, + const openscenario_interpreter::ParameterDistribution & scenario_modifier_distribution) + -> std::vector +{ + std::vector derived_scripts; + + struct replace_walker : pugi::xml_tree_walker + { + openscenario_interpreter::ParameterListSharedPtr parameter_list; + + virtual bool for_each(pugi::xml_node & node) + { + for (pugi::xml_attribute_iterator attribute_iter = node.attributes_begin(); + attribute_iter != node.attributes_end(); ++attribute_iter) { + for (const auto & parameter : *parameter_list) { + attribute_iter->set_value(regex_replace( + attribute_iter->as_string(), std::regex(parameter.first), + parameter.second.as()) + .c_str()); + } + } + return true; + } + }; + + replace_walker walker; + + for (const auto & parameter_list : scenario_modifier_distribution | boost::adaptors::indexed()) { + pugi::xml_document derived_script; + + derived_script.reset(base_scenario_doc); // deep copy + + // overwrite with scenario modifiers + walker.parameter_list = parameter_list.value(); + derived_script.traverse(walker); + + derived_scripts.push_back(std::forward(derived_script)); + } + + return derived_scripts; +} +} // namespace openscenario_preprocessor From 0b255aba3dfede5341ef03c2607a66099c2f4cb6 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 28 Apr 2023 19:36:39 +0900 Subject: [PATCH 16/52] feat(openscenario_preprocessor): add deriver class --- .../openscenario_preprocessor/deriver.hpp | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp new file mode 100644 index 00000000000..f98da37ce59 --- /dev/null +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp @@ -0,0 +1,77 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPENSCENARIO_PREPROCESSOR__DERIVER_HPP_ +#define OPENSCENARIO_PREPROCESSOR__DERIVER_HPP_ + +#include +#include +#include +#include +#include + +namespace openscenario_preprocessor +{ + +class Deriver +{ +public: + Deriver() {} + auto operator()(boost::filesystem::path path, bool check_scenario_path = true) + -> openscenario_interpreter::ParameterDistribution + { + using openscenario_interpreter::OpenScenario; + using openscenario_interpreter::ParameterValueDistribution; + using openscenario_interpreter::ParameterValueDistributionDefinition; + + validate(path); + + if (OpenScenario script{path}; script.category.is()) { + auto & parameter_value_distribution = script.category.as(); + scenario_file_path = parameter_value_distribution.scenario_file.filepath; + + if (not check_scenario_path || boost::filesystem::exists(scenario_file_path)) { + if (check_scenario_path) { + validate(scenario_file_path); + } + + scenario_file_doc.load_file(scenario_file_path.c_str()); + + return parameter_value_distribution.derive(); + } else { + std::stringstream what; + what << "Scenario file " << std::quoted(scenario_file_path.string()) + << " described in ParameterDistributionDefinition file " << std::quoted(path.string()) + << " does not exist"; + throw std::runtime_error(what.str()); + } + } else { + return {}; + } + } + + auto get_doc() -> pugi::xml_document & { return scenario_file_doc; } + + auto get_scenario_path() -> boost::filesystem::path { return scenario_file_path; } + +private: + pugi::xml_document scenario_file_doc; + + openscenario_validator::OpenSCENARIOValidator validate; + + boost::filesystem::path scenario_file_path; +}; +} // namespace openscenario_preprocessor + +#endif //OPENSCENARIO_PREPROCESSOR__DERIVER_HPP_ From 1c5fb620ea953a705a351cfd4bd60f593f5176e3 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 28 Apr 2023 19:37:03 +0900 Subject: [PATCH 17/52] feat(openscenario_preprocessor): binary first version --- .../openscenario_preprocessor.hpp | 22 ++- .../src/openscenario_preprocessor.cpp | 130 ++++++++++++------ .../src/openscenario_preprocessor_command.cpp | 129 +++++++++++++++-- .../src/openscenario_preprocessor_node.cpp | 11 +- 4 files changed, 222 insertions(+), 70 deletions(-) diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp index 2f8a99293df..b6dc9f3b067 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -38,25 +39,38 @@ struct Scenario double frame_rate; }; +enum class ScenarioFormat { + t4v2, + xosc, +}; + +std::istream & operator>>(std::istream & is, ScenarioFormat & format); + class Preprocessor { public: explicit Preprocessor(const boost::filesystem::path & output_directory) - : validate(), output_directory(output_directory) + : output_directory(output_directory) { if (not boost::filesystem::exists(output_directory)) { boost::filesystem::create_directories(output_directory); } } - void preprocessScenario(const Scenario &); + void preprocessScenario( + const boost::filesystem::path & scenario_path, + ScenarioFormat output_format = ScenarioFormat::xosc); + + void generateDerivedScenarioFromDistribution( + openscenario_interpreter::ParameterDistribution & distribution, const boost::filesystem::path & path, + ScenarioFormat output_format); protected: - std::queue preprocessed_scenarios; + std::queue preprocessed_scenarios; std::mutex preprocessed_scenarios_mutex; - openscenario_validator::OpenSCENARIOValidator validate; + Deriver derive; boost::filesystem::path output_directory; }; diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index 4bcb30bfc4f..e2a78879d5f 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -18,71 +18,111 @@ #include #include #include +#include +#include namespace openscenario_preprocessor { -void Preprocessor::preprocessScenario(const Scenario & scenario) +void Preprocessor::preprocessScenario( + const boost::filesystem::path & scenario_path, ScenarioFormat output_format) { using openscenario_interpreter::OpenScenario; using openscenario_interpreter::ParameterValueDistribution; using openscenario_interpreter::ParameterValueDistributionDefinition; - validate(scenario.path); - - if (OpenScenario script{scenario.path}; - script.category.is()) { - auto & parameter_value_distribution = script.category.as(); - auto scenario_file_path = parameter_value_distribution.scenario_file.filepath; + auto distribution = derive(scenario_path); + if (distribution.empty()) { + preprocessed_scenarios.push(scenario_path); // normal scenario + } else { + auto base_scenario_path = derive.get_scenario_path(); + generateDerivedScenarioFromDistribution(distribution, base_scenario_path, output_format); + } +} - if (boost::filesystem::exists(scenario_file_path)) { - validate(scenario_file_path); +void Preprocessor::generateDerivedScenarioFromDistribution( + openscenario_interpreter::ParameterDistribution & distribution, + const boost::filesystem::path & path, ScenarioFormat output_format) +{ +// std::cout << "generateDerivedScenarioFromDistribution" << std::endl; + for (const auto & parameter_list : distribution | boost::adaptors::indexed()) { + pugi::xml_document derived_script; - pugi::xml_document scenario_file_doc; - scenario_file_doc.load_file(scenario_file_path.c_str()); + derived_script.reset(derive.get_doc()); // deep copy - auto p = parameter_value_distribution.derive(); + auto parameter_declarations = + derived_script.document_element() + .select_node(pugi::xpath_query{"/OpenSCENARIO/ParameterDeclarations"}) + .node(); - for (const auto & parameter_list : p | boost::adaptors::indexed()) { - pugi::xml_document derived_script; + // embedding parameter values + for (const auto & [name, value] : *parameter_list.value()) { + if ( + auto parameter_node = + parameter_declarations.find_child_by_attribute("name", name.c_str())) { + parameter_node.attribute("value").set_value( + boost::lexical_cast(value).c_str()); + } else { + std::cerr << "Parameter " << std::quoted(name) << " is not declared in scenario " + << std::quoted(derive.get_scenario_path().string()) << ", so ignore it." + << std::endl; + } + } - derived_script.reset(scenario_file_doc); // deep copy + const auto derived_scenario_path_xosc = + output_directory / (path.stem().string() + "." + std::to_string(parameter_list.index()) + + path.extension().string()); - auto parameter_declarations = - derived_script.document_element() - .select_node(pugi::xpath_query{"/OpenSCENARIO/ParameterDeclarations"}) - .node(); + derived_script.save_file(derived_scenario_path_xosc.c_str()); - // embedding parameter values - for (const auto & [name, value] : *parameter_list.value()) { - if ( - auto parameter_node = - parameter_declarations.find_child_by_attribute("name", name.c_str())) { - parameter_node.attribute("value").set_value( - boost::lexical_cast(value).c_str()); - } else { - std::cerr << "Parameter " << std::quoted(name) << " is not declared in scenario " - << std::quoted(scenario_file_path.string()) << ", so ignore it." << std::endl; - } - } + try { + auto derived_scenario_path = [&]() { + if (output_format == ScenarioFormat::t4v2) { + std::cout << "convert to json : " << derived_scenario_path_xosc.c_str() << std::endl; - const auto derived_scenario_path = - output_directory / - (scenario.path.stem().string() + "." + std::to_string(parameter_list.index()) + - scenario.path.extension().string()); + pugi::xml_document derived_scenario_xml; + derived_scenario_xml.load_file(derived_scenario_path_xosc.c_str()); + auto root_xml = derived_scenario_xml.root(); - derived_script.save_file(derived_scenario_path.c_str()); + auto derived_scenario_json = tojson::pugixml2json(root_xml); +// derived_scenario_xml.print(std::cout); + std::cout << derived_scenario_json << std::endl; + std::cout << "finish convert to json" << std::endl; + const auto derived_scenario_path_t4v2 = + output_directory / + (path.stem().string() + "." + std::to_string(parameter_list.index()) + ".yaml"); +// std::cout << "save as t4v2 scenario" << std::endl; - preprocessed_scenarios.emplace(derived_scenario_path, scenario.expect, scenario.frame_rate); - } - } else { - std::stringstream what; - what << "Scenario file " << std::quoted(scenario_file_path.string()) - << " described in ParameterDistributionDefinition file " - << std::quoted(scenario.path.string()) << " does not exist"; - throw std::runtime_error(what.str()); + std::ofstream derived_scenario_yaml{derived_scenario_path_t4v2}; + derived_scenario_yaml << tojson::emitters::toyaml(derived_scenario_json); +// std::cout << "Generated " << tojson::emitters::toyaml(derived_scenario_json) << std::endl; + derived_scenario_yaml.close(); + return derived_scenario_path_t4v2; + } else { + return derived_scenario_path_xosc; + } + }(); + preprocessed_scenarios.emplace(derived_scenario_path); + } catch (rapidxml::parse_error & ex) { + std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() << ", " + << ex.where() << std::endl; + // return 1; } + } +} + +std::istream & operator>>(std::istream & is, ScenarioFormat & format) +{ + std::string token; + is >> token; + if (token == "t4v2") { + format = ScenarioFormat::t4v2; + } else if (token == "xosc") { + format = ScenarioFormat::xosc; } else { - preprocessed_scenarios.push(scenario); // normal scenario + std::stringstream what; + what << token << " is invalid scenario format. Please specify t4v2 or xosc as scenario format"; + throw std::runtime_error(what.str()); } + return is; } } // namespace openscenario_preprocessor diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index eef29e84f79..6dbba393d0e 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -13,7 +13,55 @@ // limitations under the License. #include +#include +#include #include +#include +#include + +const std::string_view template_scenario = R"###( + + + + + + + + + + + + + +)###"; + +auto create_parameter_value_distribution_from_json( + const boost::filesystem::path & scenario_path, const nlohmann::json & json_parameters) +{ + pugi::xml_document script; + script.load_string(template_scenario.data()); + + script.document_element() + .select_node(pugi::xpath_query{"/OpenSCENARIO/ParameterValueDistribution/ScenarioFile"}) + .node() + .attribute("filepath") + .set_value(scenario_path.c_str()); + + auto value_set_node = + script.document_element() + .select_node(pugi::xpath_query{ + "/OpenSCENARIO/ParameterValueDistribution/Deterministic/" + "DeterministicMultiParameterDistribution/ValueSetDistribution/ParameterValueSet"}) + .node(); + + for (auto const & json_parameter : json_parameters.items()) { + auto parameter_assignment_node = value_set_node.append_child("ParameterAssignment"); + parameter_assignment_node.append_attribute("parameterRef") = json_parameter.key().c_str(); + parameter_assignment_node.append_attribute("value") = + json_parameter.value().get().c_str(); + } + return script; +} int main(const int argc, char const * const * const argv) { @@ -23,37 +71,88 @@ int main(const int argc, char const * const * const argv) // -o -p param.json -s scenario.yaml description.add_options()( - "output-directory,o", value()->default_value("/tmp/openscenario_preprocessor"), + "output-directory,o", + value()->default_value("/tmp/openscenario_preprocessor/derived"), "path of output directory")( - "parameters,p", value()->default_value(""), "parameters in json format")( + "format,f", value()->multitoken(), + "output scenario format (t4v2 / xosc)")( + "parameters,p", value()->default_value("null"), "parameters in json format")( "scenario,s", value(), "path of scenario file")("help,H", "help"); - std::cout << "setup description" << std::endl; - variables_map vm; store(parse_command_line(argc, argv, description), vm); notify(vm); - std::cout << "setup variables map" << std::endl; - auto output_directory_option = boost::filesystem::path(vm["output-directory"].as()); + auto format_option = vm["format"].as(); auto parameters_option = boost::filesystem::path(vm["parameters"].as()); auto scenario_option = boost::filesystem::path(vm["scenario"].as()); - std::cout << "get options" << std::endl; + auto scenario_path = boost::filesystem::path(scenario_option); + + boost::filesystem::path tmp_output_directory = "/tmp/openscenario_preprocessor"; + if(not boost::filesystem::exists(tmp_output_directory)){ + boost::filesystem::create_directories(tmp_output_directory); + } + + std::vector xosc_scenario_paths; + + // preprocess t4v2 format and convert to xosc scenarios + if (scenario_path.extension() == ".yaml" or scenario_path.extension() == ".yml") { + openscenario_preprocessor::T4V2 t4v2; + auto xosc_string_scenarios = t4v2.deriveToXoscStringScenarios(scenario_path); - openscenario_preprocessor::Preprocessor preprocessor(output_directory_option); - std::cout << "create preprocessor" << std::endl; + boost::filesystem::path t4v2_output_directory = "/tmp/openscenario_preprocessor/t4v2_derived"; + if(not boost::filesystem::exists(t4v2_output_directory)){ + boost::filesystem::create_directories(t4v2_output_directory); + } + for (const auto & xosc_string_scenario : xosc_string_scenarios | boost::adaptors::indexed()) { +// std::cout << xosc_string_scenario.value() << std::endl; + boost::filesystem::path xosc_scenario_path = + t4v2_output_directory / (std::to_string(xosc_string_scenario.index()) + ".xosc"); + std::ofstream ofs(xosc_scenario_path.c_str()); + ofs << xosc_string_scenario.value().c_str(); + ofs.close(); + xosc_scenario_paths.push_back(xosc_scenario_path); + } +// std::cout << "finish writing xosc scenarios from t4v2 format" << std::endl; + } else { + xosc_scenario_paths.push_back(scenario_path); + } - openscenario_preprocessor::Scenario scenario; - scenario.path = scenario_option; - scenario.expect = 1; - scenario.frame_rate = 30.0; + // derive for given parameters + if (parameters_option != "null") { + for (auto scenario_path : xosc_scenario_paths) { + auto parameter_value_distribution = create_parameter_value_distribution_from_json( + scenario_path, nlohmann::json::parse(parameters_option.c_str())); - preprocessor.preprocessScenario(scenario); + parameter_value_distribution.save_file( + "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc"); - std::cout << "preprocess scenario" << std::endl; + openscenario_preprocessor::Preprocessor preprocessor(output_directory_option); +// try { + preprocessor.preprocessScenario( + "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc", format_option); +// }catch (rapidxml::parse_error & ex){ +// std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() +// << ", " << ex.where() << std::endl; +//// return 1; +// } + // } catch (std::runtime_error & ex) { + // std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() + // << std::endl; + // return 1; + // }catch (std::exception & ex) { + // std::cerr << "[Error] unknown error occured during deriving scenario : " << ex.what() + // << std::endl; + // std::cerr << "Please contact developers of this software." << std::endl; + // return 1; + // } + } + } else { + // TODO + } return 0; } diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp index 124561d2ffe..4d0cb475154 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp @@ -36,14 +36,13 @@ class PreprocessorNode : public rclcpp::Node, public openscenario_preprocessor:: openscenario_preprocessor_msgs::srv::Load::Response::SharedPtr response) -> void { auto lock = std::lock_guard(preprocessed_scenarios_mutex); try { - preprocessScenario(openscenario_preprocessor::Scenario( - request->path, request->expect, request->frame_rate)); + preprocessScenario(request->path); response->has_succeeded = true; response->message = "success"; } catch (std::exception & e) { response->has_succeeded = false; response->message = e.what(); - std::queue().swap(preprocessed_scenarios); +// std::queue().swap(preprocessed_scenarios); } })), derive_server(create_service( @@ -55,9 +54,9 @@ class PreprocessorNode : public rclcpp::Node, public openscenario_preprocessor:: if (preprocessed_scenarios.empty()) { response->path = "no output"; } else { - response->path = preprocessed_scenarios.front().path.string(); - response->expect = preprocessed_scenarios.front().expect; - response->frame_rate = preprocessed_scenarios.front().frame_rate; + response->path = preprocessed_scenarios.front().string(); + response->expect = 1; + response->frame_rate = 30.0; preprocessed_scenarios.pop(); } })), From b565c33f50c8ae9fdac2f569f48609fa52dc2d40 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 28 Apr 2023 19:37:50 +0900 Subject: [PATCH 18/52] chore(openscenario_preprocessor): apply linter --- .../openscenario_preprocessor.hpp | 4 +-- .../openscenario_preprocessor/tojson.hpp | 4 +-- .../src/openscenario_preprocessor.cpp | 8 +++--- .../src/openscenario_preprocessor_command.cpp | 25 +++++++++---------- .../src/openscenario_preprocessor_node.cpp | 2 +- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp index b6dc9f3b067..dde85e2a845 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp @@ -62,8 +62,8 @@ class Preprocessor ScenarioFormat output_format = ScenarioFormat::xosc); void generateDerivedScenarioFromDistribution( - openscenario_interpreter::ParameterDistribution & distribution, const boost::filesystem::path & path, - ScenarioFormat output_format); + openscenario_interpreter::ParameterDistribution & distribution, + const boost::filesystem::path & path, ScenarioFormat output_format); protected: std::queue preprocessed_scenarios; diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp index 435c56dcb4d..e71506cb97f 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp @@ -130,7 +130,7 @@ inline json pugixml2json(const pugi::xml_node & root) if (child == child.parent().first_child()) n[key].emplace_back(pugixml2json(child)); else -// n[key].emplace_back(child.value()); + // n[key].emplace_back(child.value()); n["@text"] = child.value(); // toyaml through the attributes @@ -143,7 +143,7 @@ inline json pugixml2json(const pugi::xml_node & root) if (child == child.parent().first_child()) j[key] = pugixml2json(child); else -// j[key].emplace_back(child.value()); + // j[key].emplace_back(child.value()); j["@text"] = child.value(); // toyaml through the attributes for (auto & attr : child.attributes()) { diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index e2a78879d5f..7a241259f35 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -43,7 +43,7 @@ void Preprocessor::generateDerivedScenarioFromDistribution( openscenario_interpreter::ParameterDistribution & distribution, const boost::filesystem::path & path, ScenarioFormat output_format) { -// std::cout << "generateDerivedScenarioFromDistribution" << std::endl; + // std::cout << "generateDerivedScenarioFromDistribution" << std::endl; for (const auto & parameter_list : distribution | boost::adaptors::indexed()) { pugi::xml_document derived_script; @@ -84,17 +84,17 @@ void Preprocessor::generateDerivedScenarioFromDistribution( auto root_xml = derived_scenario_xml.root(); auto derived_scenario_json = tojson::pugixml2json(root_xml); -// derived_scenario_xml.print(std::cout); + // derived_scenario_xml.print(std::cout); std::cout << derived_scenario_json << std::endl; std::cout << "finish convert to json" << std::endl; const auto derived_scenario_path_t4v2 = output_directory / (path.stem().string() + "." + std::to_string(parameter_list.index()) + ".yaml"); -// std::cout << "save as t4v2 scenario" << std::endl; + // std::cout << "save as t4v2 scenario" << std::endl; std::ofstream derived_scenario_yaml{derived_scenario_path_t4v2}; derived_scenario_yaml << tojson::emitters::toyaml(derived_scenario_json); -// std::cout << "Generated " << tojson::emitters::toyaml(derived_scenario_json) << std::endl; + // std::cout << "Generated " << tojson::emitters::toyaml(derived_scenario_json) << std::endl; derived_scenario_yaml.close(); return derived_scenario_path_t4v2; } else { diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 6dbba393d0e..2341a0b2cd9 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -91,7 +91,7 @@ int main(const int argc, char const * const * const argv) auto scenario_path = boost::filesystem::path(scenario_option); boost::filesystem::path tmp_output_directory = "/tmp/openscenario_preprocessor"; - if(not boost::filesystem::exists(tmp_output_directory)){ + if (not boost::filesystem::exists(tmp_output_directory)) { boost::filesystem::create_directories(tmp_output_directory); } @@ -102,13 +102,12 @@ int main(const int argc, char const * const * const argv) openscenario_preprocessor::T4V2 t4v2; auto xosc_string_scenarios = t4v2.deriveToXoscStringScenarios(scenario_path); - boost::filesystem::path t4v2_output_directory = "/tmp/openscenario_preprocessor/t4v2_derived"; - if(not boost::filesystem::exists(t4v2_output_directory)){ + if (not boost::filesystem::exists(t4v2_output_directory)) { boost::filesystem::create_directories(t4v2_output_directory); } for (const auto & xosc_string_scenario : xosc_string_scenarios | boost::adaptors::indexed()) { -// std::cout << xosc_string_scenario.value() << std::endl; + // std::cout << xosc_string_scenario.value() << std::endl; boost::filesystem::path xosc_scenario_path = t4v2_output_directory / (std::to_string(xosc_string_scenario.index()) + ".xosc"); std::ofstream ofs(xosc_scenario_path.c_str()); @@ -116,7 +115,7 @@ int main(const int argc, char const * const * const argv) ofs.close(); xosc_scenario_paths.push_back(xosc_scenario_path); } -// std::cout << "finish writing xosc scenarios from t4v2 format" << std::endl; + // std::cout << "finish writing xosc scenarios from t4v2 format" << std::endl; } else { xosc_scenario_paths.push_back(scenario_path); } @@ -132,14 +131,14 @@ int main(const int argc, char const * const * const argv) openscenario_preprocessor::Preprocessor preprocessor(output_directory_option); -// try { - preprocessor.preprocessScenario( - "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc", format_option); -// }catch (rapidxml::parse_error & ex){ -// std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() -// << ", " << ex.where() << std::endl; -//// return 1; -// } + // try { + preprocessor.preprocessScenario( + "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc", format_option); + // }catch (rapidxml::parse_error & ex){ + // std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() + // << ", " << ex.where() << std::endl; + //// return 1; + // } // } catch (std::runtime_error & ex) { // std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() // << std::endl; diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp index 4d0cb475154..f82f1e1ead2 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_node.cpp @@ -42,7 +42,7 @@ class PreprocessorNode : public rclcpp::Node, public openscenario_preprocessor:: } catch (std::exception & e) { response->has_succeeded = false; response->message = e.what(); -// std::queue().swap(preprocessed_scenarios); + // std::queue().swap(preprocessed_scenarios); } })), derive_server(create_service( From ef25041c9bfa9acbc11921bfd2542428104d6f60 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 1 May 2023 10:29:42 +0900 Subject: [PATCH 19/52] feat(openscenario_preprocessor): add yaml_xml.hpp --- .../openscenario_preprocessor/yaml_xml.hpp | 232 ++++++++++++++++++ .../src/openscenario_preprocessor.cpp | 1 + 2 files changed, 233 insertions(+) create mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp new file mode 100644 index 00000000000..47b84006519 --- /dev/null +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp @@ -0,0 +1,232 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef OPENSCENARIO_PREPROCESSOR__YAML_XML_HPP_ +#define OPENSCENARIO_PREPROCESSOR__YAML_XML_HPP_ + +#include + +#include +#include +#include + +namespace YAML +{ +template <> +struct convert +{ + static Node encode(const pugi::xml_node & xml) + { + Node node; + return node; + } + // + // static bool decode(const Node & yaml, const Node & parent_yaml, pugi::xml_node & xml) + // { + // switch (yaml.Type()) { + // case NodeType::Null: + // break; + // case NodeType::Scalar: + // + // // TODO: put in attribute + // // xml_node.append_attribute(pugi::node_) + // break; + // case NodeType::Sequence: + // for (const auto & node : yaml) { + // xml.append_child(parent_yaml.); + // convertYAMLtoXML(node, yaml, xml_node.append_child(pugi::node_element, "TODO")); + // } + // break; + // case NodeType::Map: + // for (YAML::const_iterator iter = yaml.begin(); iter != yaml.end(); ++iter) { + // auto value = iter->second; + // if (value.IsScalar()) { + // xml.append_attribute(iter->first.as().c_str()) = + // value.as().c_str(); + // } else { + // decode(yaml, xml); + // } + // } + // case NodeType::Undefined: + // break; + // } + // rhs.x = node[0].as(); + // rhs.y = node[1].as(); + // rhs.z = node[2].as(); + // return true; + // } +}; +} // namespace YAML + +namespace openscenario_preprocessor +{ +namespace yaml_xml +{ + +class YamlXml +{ +public: + YamlXml() = default; + + ~YamlXml() = default; + + void loadYAMLfile(const std::filesystem::path & file_path) + { + std::ifstream ifs(file_path); + *yaml_node_ = YAML::Load(ifs); + xml_document_ = std::nullopt; + } + + void loadXMLfile(const std::filesystem::path & file_path) + { + *xml_document_ = pugi::xml_document(); + xml_document_->load_file(file_path.c_str()); + yaml_node_ = std::nullopt; + } + + void loadYAMLString(const std::string & yaml_string) + { + *yaml_node_ = YAML::Load(yaml_string); + xml_document_ = std::nullopt; + } + + void loadXMLString(const std::string & xml_string) + { + *xml_document_ = pugi::xml_document(); + xml_document_->load_string(xml_string.c_str()); + yaml_node_ = std::nullopt; + } + + void convertYAMLtoXML() + { + if (yaml_node_) { + *xml_document_ = pugi::xml_document(); + *xml_document_ = YAML::convert::encode(*yaml_node_); + } + } + + void convertXMLtoYAML() + { + if (xml_document_) { + *yaml_node_ = YAML::Load(pugi::as_utf8(xml_document_->child_value())); + } + } + + void saveYAMLFile(const std::filesystem::path & file_path) + { + if (not yaml_node_) { + convertXMLtoYAML(); + } + std::ofstream ofs(file_path); + ofs << *yaml_node_; + ofs.close(); + } + + void saveXMLFile(const std::filesystem::path & file_path) + { + if (not xml_document_) { + convertYAMLtoXML(); + } + xml_document_->save_file(file_path.c_str()); + } + +private: + void convertXMLtoYAMLImpl(const pugi::xml_node & xml, YAML::Node & yaml) + { + std::map count; + for (const auto & child : xml.children()) { + if (not count[child.name()]) { + count[child.name()] = 1; + } else { + count[child.name()]++; + } + } + + YAML::Emitter out; + for (const auto & child : xml.children()) { + if (count[child.name()] > 1) { + if (yaml[child.name()].IsNull()) { + // first node + out << YAML::BeginMap; + out << YAML::Key << child.name(); + out << YAML::Value << YAML::BeginSeq; + + }else{ + + } + } + } + } && + std::find(yaml.begin(), yaml.end(), [&](YAML::const_iterator & iter) -> bool { + return iter->first.as() == child.name(); + }) == yaml.end()) + { + YAML::Emitter out; + out << YAML::BeginMap; + out << YAML::Key << child.name(); + out << YAML::Value << YAML::BeginSeq; + } + yaml[child.name()] = YAML::; + convertXMLtoYAMLImpl(child, yaml[child.name()][count[child.name()] - 1]); +} + +for (const auto & child : xml.children()) +{ + if (child.type() == pugi::node_pcdata) { + yaml = child.value(); + } else { + yaml = YAML::Node(); + convertXMLtoYAMLImpl(child, yaml); + } +} +} +void convertYAMLtoXMLImpl(const YAML::Node & yaml_node, pugi::xml_node & xml_node) +{ + // using YAML::NodeType; + // switch (yaml_node.Type()) { + // case NodeType::Null: + // break; + // case NodeType::Scalar: + // // TODO: put in attribute + // // xml_node.append_attribute(pugi::node_) + // break; + // case NodeType::Sequence: + // for (const auto & node : yaml_node) { + // convertYAMLtoXML(node, xml_node.append_child(pugi::node_element, "TODO")); + // } + // break; + // case NodeType::Map: + // yaml_node->fir break; + // case NodeType::Undefined: + // break; + // } + // if (yaml_node.IsScalar()) { + // xml_node.append_child(pugi::node_pcdata).set_value(yaml_node.as().c_str()); + // } else if (yaml_node.IsSequence()) { + // xml_node.append_child(yaml_node.) + // + // } else if (yaml_node.IsMap()) { + // for (const auto & node : yaml_node) { + // convertYAMLtoXML(node.second, xml_node.append_child(node.first.as().c_str())); + // } + // } +} +std::optional yaml_node_ = std::nullopt; + +std::optional xml_document_ = std::nullopt; +}; +} // namespace yaml_xml +} // namespace openscenario_preprocessor + +#endif // OPENSCENARIO_PREPROCESSOR__YAML_XML_HPP_ diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index 7a241259f35..ee49946cae4 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace openscenario_preprocessor From 49a196279f1a1d6518f1eb3c51a5136d7b71abd8 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 1 May 2023 14:32:03 +0900 Subject: [PATCH 20/52] feat(openscenario_preprocessor): support yaml output from command --- .../openscenario_preprocessor.hpp | 4 + .../openscenario_preprocessor/yaml_xml.hpp | 232 ------------------ .../src/openscenario_preprocessor.cpp | 85 +++++-- 3 files changed, 75 insertions(+), 246 deletions(-) delete mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp index dde85e2a845..ef09186cb01 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp @@ -15,6 +15,8 @@ #ifndef OPENSCENARIO_PREPROCESSOR__OPENSCENARIO_PREPROCESSOR_HPP_ #define OPENSCENARIO_PREPROCESSOR__OPENSCENARIO_PREPROCESSOR_HPP_ +#include + #include #include #include @@ -65,6 +67,8 @@ class Preprocessor openscenario_interpreter::ParameterDistribution & distribution, const boost::filesystem::path & path, ScenarioFormat output_format); + void convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & emitter); + protected: std::queue preprocessed_scenarios; diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp deleted file mode 100644 index 47b84006519..00000000000 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/yaml_xml.hpp +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2015 TIER IV, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPENSCENARIO_PREPROCESSOR__YAML_XML_HPP_ -#define OPENSCENARIO_PREPROCESSOR__YAML_XML_HPP_ - -#include - -#include -#include -#include - -namespace YAML -{ -template <> -struct convert -{ - static Node encode(const pugi::xml_node & xml) - { - Node node; - return node; - } - // - // static bool decode(const Node & yaml, const Node & parent_yaml, pugi::xml_node & xml) - // { - // switch (yaml.Type()) { - // case NodeType::Null: - // break; - // case NodeType::Scalar: - // - // // TODO: put in attribute - // // xml_node.append_attribute(pugi::node_) - // break; - // case NodeType::Sequence: - // for (const auto & node : yaml) { - // xml.append_child(parent_yaml.); - // convertYAMLtoXML(node, yaml, xml_node.append_child(pugi::node_element, "TODO")); - // } - // break; - // case NodeType::Map: - // for (YAML::const_iterator iter = yaml.begin(); iter != yaml.end(); ++iter) { - // auto value = iter->second; - // if (value.IsScalar()) { - // xml.append_attribute(iter->first.as().c_str()) = - // value.as().c_str(); - // } else { - // decode(yaml, xml); - // } - // } - // case NodeType::Undefined: - // break; - // } - // rhs.x = node[0].as(); - // rhs.y = node[1].as(); - // rhs.z = node[2].as(); - // return true; - // } -}; -} // namespace YAML - -namespace openscenario_preprocessor -{ -namespace yaml_xml -{ - -class YamlXml -{ -public: - YamlXml() = default; - - ~YamlXml() = default; - - void loadYAMLfile(const std::filesystem::path & file_path) - { - std::ifstream ifs(file_path); - *yaml_node_ = YAML::Load(ifs); - xml_document_ = std::nullopt; - } - - void loadXMLfile(const std::filesystem::path & file_path) - { - *xml_document_ = pugi::xml_document(); - xml_document_->load_file(file_path.c_str()); - yaml_node_ = std::nullopt; - } - - void loadYAMLString(const std::string & yaml_string) - { - *yaml_node_ = YAML::Load(yaml_string); - xml_document_ = std::nullopt; - } - - void loadXMLString(const std::string & xml_string) - { - *xml_document_ = pugi::xml_document(); - xml_document_->load_string(xml_string.c_str()); - yaml_node_ = std::nullopt; - } - - void convertYAMLtoXML() - { - if (yaml_node_) { - *xml_document_ = pugi::xml_document(); - *xml_document_ = YAML::convert::encode(*yaml_node_); - } - } - - void convertXMLtoYAML() - { - if (xml_document_) { - *yaml_node_ = YAML::Load(pugi::as_utf8(xml_document_->child_value())); - } - } - - void saveYAMLFile(const std::filesystem::path & file_path) - { - if (not yaml_node_) { - convertXMLtoYAML(); - } - std::ofstream ofs(file_path); - ofs << *yaml_node_; - ofs.close(); - } - - void saveXMLFile(const std::filesystem::path & file_path) - { - if (not xml_document_) { - convertYAMLtoXML(); - } - xml_document_->save_file(file_path.c_str()); - } - -private: - void convertXMLtoYAMLImpl(const pugi::xml_node & xml, YAML::Node & yaml) - { - std::map count; - for (const auto & child : xml.children()) { - if (not count[child.name()]) { - count[child.name()] = 1; - } else { - count[child.name()]++; - } - } - - YAML::Emitter out; - for (const auto & child : xml.children()) { - if (count[child.name()] > 1) { - if (yaml[child.name()].IsNull()) { - // first node - out << YAML::BeginMap; - out << YAML::Key << child.name(); - out << YAML::Value << YAML::BeginSeq; - - }else{ - - } - } - } - } && - std::find(yaml.begin(), yaml.end(), [&](YAML::const_iterator & iter) -> bool { - return iter->first.as() == child.name(); - }) == yaml.end()) - { - YAML::Emitter out; - out << YAML::BeginMap; - out << YAML::Key << child.name(); - out << YAML::Value << YAML::BeginSeq; - } - yaml[child.name()] = YAML::; - convertXMLtoYAMLImpl(child, yaml[child.name()][count[child.name()] - 1]); -} - -for (const auto & child : xml.children()) -{ - if (child.type() == pugi::node_pcdata) { - yaml = child.value(); - } else { - yaml = YAML::Node(); - convertXMLtoYAMLImpl(child, yaml); - } -} -} -void convertYAMLtoXMLImpl(const YAML::Node & yaml_node, pugi::xml_node & xml_node) -{ - // using YAML::NodeType; - // switch (yaml_node.Type()) { - // case NodeType::Null: - // break; - // case NodeType::Scalar: - // // TODO: put in attribute - // // xml_node.append_attribute(pugi::node_) - // break; - // case NodeType::Sequence: - // for (const auto & node : yaml_node) { - // convertYAMLtoXML(node, xml_node.append_child(pugi::node_element, "TODO")); - // } - // break; - // case NodeType::Map: - // yaml_node->fir break; - // case NodeType::Undefined: - // break; - // } - // if (yaml_node.IsScalar()) { - // xml_node.append_child(pugi::node_pcdata).set_value(yaml_node.as().c_str()); - // } else if (yaml_node.IsSequence()) { - // xml_node.append_child(yaml_node.) - // - // } else if (yaml_node.IsMap()) { - // for (const auto & node : yaml_node) { - // convertYAMLtoXML(node.second, xml_node.append_child(node.first.as().c_str())); - // } - // } -} -std::optional yaml_node_ = std::nullopt; - -std::optional xml_document_ = std::nullopt; -}; -} // namespace yaml_xml -} // namespace openscenario_preprocessor - -#endif // OPENSCENARIO_PREPROCESSOR__YAML_XML_HPP_ diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index ee49946cae4..0a86debefb3 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -73,32 +73,22 @@ void Preprocessor::generateDerivedScenarioFromDistribution( output_directory / (path.stem().string() + "." + std::to_string(parameter_list.index()) + path.extension().string()); - derived_script.save_file(derived_scenario_path_xosc.c_str()); - try { auto derived_scenario_path = [&]() { if (output_format == ScenarioFormat::t4v2) { - std::cout << "convert to json : " << derived_scenario_path_xosc.c_str() << std::endl; - - pugi::xml_document derived_scenario_xml; - derived_scenario_xml.load_file(derived_scenario_path_xosc.c_str()); - auto root_xml = derived_scenario_xml.root(); + YAML::Emitter yaml_emitter; + convertXMLtoYAML(derived_script, yaml_emitter); - auto derived_scenario_json = tojson::pugixml2json(root_xml); - // derived_scenario_xml.print(std::cout); - std::cout << derived_scenario_json << std::endl; - std::cout << "finish convert to json" << std::endl; const auto derived_scenario_path_t4v2 = output_directory / (path.stem().string() + "." + std::to_string(parameter_list.index()) + ".yaml"); - // std::cout << "save as t4v2 scenario" << std::endl; std::ofstream derived_scenario_yaml{derived_scenario_path_t4v2}; - derived_scenario_yaml << tojson::emitters::toyaml(derived_scenario_json); - // std::cout << "Generated " << tojson::emitters::toyaml(derived_scenario_json) << std::endl; + derived_scenario_yaml << yaml_emitter.c_str(); derived_scenario_yaml.close(); return derived_scenario_path_t4v2; } else { + derived_script.save_file(derived_scenario_path_xosc.c_str()); return derived_scenario_path_xosc; } }(); @@ -111,6 +101,73 @@ void Preprocessor::generateDerivedScenarioFromDistribution( } } +void Preprocessor::convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & emitter) +{ + if (xml.attributes().empty() && xml.children().empty()) { + emitter << ""; + return; + } + + // the map of node name and {total count, used count} + std::map> count; + for (const auto & child : xml.children()) { + if (count.find(child.name()) != count.end()) { + count[child.name()] = std::make_pair(1, 0); + } else { + count[child.name()].first++; + } + } + + // iterate attributes + if (not xml.attributes().empty()) { + emitter << YAML::BeginMap; + + for (const auto & attr : xml.attributes()) { + emitter << YAML::Key << attr.name(); + emitter << YAML::Value << YAML::SingleQuoted << attr.as_string(); + } + // end map in child element part + } + + // iterate child elements + if (not xml.children().empty()) { + if (xml.attributes().empty()) { + emitter << YAML::BeginMap; + } + + for (const auto & child : xml.children()) { + // count == 1, make single map in yaml + if (count[child.name()].first == 1) { + emitter << YAML::Key << child.name(); + emitter << YAML::Value; + convertXMLtoYAML(child, emitter); + } else { + // count > 1, make sequence in yaml + if (count[child.name()].second == 0) { + // first node + emitter << YAML::BeginSeq; + emitter << YAML::Key << child.name(); + emitter << YAML::Value; + convertXMLtoYAML(child, emitter); + count[child.name()].second++; + } else if (count[child.name()].second == count[child.name()].first - 1) { + // last node + convertXMLtoYAML(child, emitter); + emitter << YAML::EndSeq; + } else { + // middle node + convertXMLtoYAML(child, emitter); + count[child.name()].second++; + } + } + } + } + + if (not xml.attributes().empty() or not xml.children().empty()) { + emitter << YAML::EndMap; + } +} + std::istream & operator>>(std::istream & is, ScenarioFormat & format) { std::string token; From 84583cb15204ae8f2fdd47eb0a0f0ce003e8fcd6 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 1 May 2023 16:58:37 +0900 Subject: [PATCH 21/52] refactor(openscenario_preprocessor): delete tojson.hpp --- .../openscenario_preprocessor/t4v2.hpp | 3 +- .../openscenario_preprocessor/tojson.hpp | 383 ------------------ .../openscenario_preprocessor/src/t4v2.cpp | 52 ++- 3 files changed, 46 insertions(+), 392 deletions(-) delete mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp index 289d76f8cf5..2f8f0396eb4 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp @@ -17,7 +17,6 @@ #include #include -#include #include namespace openscenario_preprocessor @@ -27,6 +26,8 @@ class T4V2 public: auto deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vector; + auto loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document; + auto generateParameterValueDistributionFromScenarioModifiers(std::string scenario_modifiers_str) -> pugi::xml_document; diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp deleted file mode 100644 index e71506cb97f..00000000000 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/tojson.hpp +++ /dev/null @@ -1,383 +0,0 @@ -// MIT License -// -// Copyright (c) 2021 Mirco -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -#ifndef OPENSCENARIO_PREPROCESSOR_TOJSON_HPP -#define OPENSCENARIO_PREPROCESSOR_TOJSON_HPP - -#include - -#include -#include -#include -#include - -/* Adding declarations to make it compatible with gcc 4.7 and greater */ -#if __GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__ > 40700 -namespace rapidxml -{ -namespace internal -{ -template -inline OutIt print_children(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_attributes(OutIt out, const xml_node * node, int flags); -template -inline OutIt print_data_node(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_cdata_node(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_element_node(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_declaration_node(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_comment_node(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_doctype_node(OutIt out, const xml_node * node, int flags, int indent); -template -inline OutIt print_pi_node(OutIt out, const xml_node * node, int flags, int indent); -} // namespace internal -} // namespace rapidxml -#include -#endif - -#if __has_cpp_attribute(nodiscard) -#define TOJSON_NODISCARD [[nodiscard]] -#else -#define TOJSON_NODISCARD -#endif - -#include -using json = nlohmann::ordered_json; - -namespace tojson -{ -namespace detail -{ - -/// \todo refactor and pass nlohmann::json down by reference instead of returning it -inline json xml2json(const rapidxml::xml_node<> * root) -{ - json j{}; - std::map occurrence{}; - - for (auto * node = root->first_node(); node; node = node->next_sibling()) { - auto key = node->name(); - occurrence[key] += 1; - - // if seen current node, convert it into a list - if (occurrence[key] == 2) j[key] = {j[key]}; - - // if the node is a sequence - if (occurrence[key] > 1) { - json n{}; - if (node->first_node()) - n[key] = xml2json(node); - else - n["@text"] = node->value(); - // toyaml through the attributes - for (auto * attr = node->first_attribute(); attr; attr = attr->next_attribute()) - n[key][attr->name()] = attr->value(); - j[key].emplace_back(n[key]); - } else { - if (node->first_node()) - j[key] = xml2json(node); - else - j["@text"] = node->value(); - // toyaml through the attributes - for (auto * attr = node->first_attribute(); attr; attr = attr->next_attribute()) - j[key][attr->name()] = attr->value(); - } - } - - return j; -} - -/// \todo refactor and pass nlohmann::json down by reference instead of returning it -inline json pugixml2json(const pugi::xml_node & root) -{ - json j{}; - std::map occurrence{}; - - for (auto & child : root.children()) { - auto key = child.name(); - occurrence[key] += 1; - - // if seen current node, convert it into a list - if (occurrence[key] == 2) j[key] = json::array({j[key]}); - - // if the node is a sequence - if (occurrence[key] > 1) { - json n{}; - if (child == child.parent().first_child()) - n[key].emplace_back(pugixml2json(child)); - else - // n[key].emplace_back(child.value()); - - n["@text"] = child.value(); - // toyaml through the attributes - for (auto & attr : child.attributes()) { - n[key][attr.name()] = attr.value(); - } - - j[key].emplace_back(n[key]); - } else { - if (child == child.parent().first_child()) - j[key] = pugixml2json(child); - else - // j[key].emplace_back(child.value()); - j["@text"] = child.value(); - // toyaml through the attributes - for (auto & attr : child.attributes()) { - j[key][attr.name()] = attr.value(); - } - } - } - - return j; -} - -inline json parse_scalar(const YAML::Node & node) -{ - int i; - double d; - bool b; - std::string s; - - if (YAML::convert::decode(node, i)) return i; - if (YAML::convert::decode(node, d)) return d; - if (YAML::convert::decode(node, b)) return b; - if (YAML::convert::decode(node, s)) return s; - - return nullptr; -} - -/// \todo refactor and pass nlohmann::json down by reference instead of returning it -inline json yaml2json(const YAML::Node & root) -{ - json j{}; - - switch (root.Type()) { - case YAML::NodeType::Null: - break; - case YAML::NodeType::Scalar: - return parse_scalar(root); - case YAML::NodeType::Sequence: - for (auto && node : root) j.emplace_back(yaml2json(node)); - break; - case YAML::NodeType::Map: - for (auto && it : root) j[it.first.as()] = yaml2json(it.second); - break; - default: - break; - } - return j; -} - -/// \todo handle @text entries better -inline void toyaml(const json & j, YAML::Emitter & e) -{ - for (auto it = j.begin(); it != j.end(); ++it) { - if (it->is_object()) { - e << YAML::Key << it.key() << YAML::Value << YAML::BeginMap; - toyaml(*it, e); - e << YAML::EndMap; - } else if (it->is_array()) { - e << YAML::Key << it.key() << YAML::Value << YAML::BeginSeq; - toyaml(it.value(), e); - e << YAML::EndSeq; - } else { - if (it.key() == "@text") { - e << YAML::Value << it.value().get(); - } else { - e << YAML::Key << it.key() << YAML::Value << it.value().get(); - } - } - } -} - -// Forward declaration required here for circular dipedency. -inline void toxml(const json & j, rapidxml::xml_document<> & doc, rapidxml::xml_node<> * parent); - -inline std::string repr(const json & j) -{ - if (j.is_number()) return std::to_string(j.get()); - if (j.is_boolean()) return j.get() ? "true" : "false"; - if (j.is_number_float()) return std::to_string(j.get()); - if (j.is_string()) return j.get(); - std::runtime_error("invalid type"); - return ""; -} - -/// \todo handle @text entries better -inline void toxml( - const json & j, rapidxml::xml_document<> & doc, rapidxml::xml_node<> * parent, - const std::string & key) -{ - // std::cout << "toxml for array : " << key << std::endl; - // Not the prettiest of designs, but it works fine. - for (auto it = j.begin(); it != j.end(); ++it) { - if (it->is_object()) { - // std::cout << "object in array : " << *it << std::endl; - // std::cout << it.key() << std::endl; - // auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(it.key().data())); - auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(key.data())); - // auto * node = doc.allocate_node(rapidxml::node_element); - detail::toxml(*it, doc, node); - parent->append_node(node); - } else if (it->is_array()) { - // std::cout << "array in array" << std::endl; - detail::toxml(*it, doc, parent, key); - } else { - // std::cout << "else in array : " << std::endl; - // std::cout << it.key() << std::endl; - auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(key.data())); - node->value(doc.allocate_string(repr(it.value()).data())); - parent->append_node(node); - } - } -} - -/// \todo handle @text entries better -inline void toxml(const json & j, rapidxml::xml_document<> & doc, rapidxml::xml_node<> * parent) -{ - // std::cout << "detail::toxml 1" << std::endl; - for (auto it = j.begin(); it != j.end(); ++it) { - if (it->is_object()) { - auto * node = doc.allocate_node(rapidxml::node_element, doc.allocate_string(it.key().data())); - detail::toxml(*it, doc, node); - parent->append_node(node); - } else if (it->is_array()) { - detail::toxml(*it, doc, parent, it.key()); - } else { - // std::cout << "detail::toxml else : "<< it.key().data() << std::endl; - if (it->is_null()) { - detail::toxml(*it, doc, parent, it.key()); - - } else { - auto * node = doc.allocate_attribute(doc.allocate_string(it.key().data())); - node->value(doc.allocate_string(repr(it.value()).data())); - parent->append_attribute(node); - } - } - } -} - -} // namespace detail - -/// \brief Convert XML string to JSON. -TOJSON_NODISCARD inline json xml2json(const std::string & str) -{ - std::string replacement = "\\/"; - boost::regex re("(?)"); - std::string escaped_str = boost::regex_replace(str, re, replacement); - - json j{}; - rapidxml::xml_document<> doc{}; - doc.parse<0>(const_cast(escaped_str.data())); - - auto * root = doc.first_node(); - if (root) j[root->name()] = detail::xml2json(root); - - return j; -} - -TOJSON_NODISCARD inline json pugixml2json(pugi::xml_node & root) -{ - json j{}; - j[root.name()] = detail::pugixml2json(root); - - return j; -} - -/// \brief Convert YAML string to JSON. -TOJSON_NODISCARD inline json yaml2json(const std::string & str) -{ - YAML::Node root = YAML::Load(str); - return detail::yaml2json(root); -} - -/// \brief Load a YAML file to JSON. -TOJSON_NODISCARD inline json loadyaml(const std::string & filepath) -{ - YAML::Node root = YAML::LoadFile(filepath); - return detail::yaml2json(root); -} - -/// \brief Load XML file to JSON. -TOJSON_NODISCARD inline json loadxml(const std::string & filepath) -{ - std::ifstream file{filepath.data()}; - std::string str{std::istream_iterator(file), std::istream_iterator()}; - // std::cout << "loaded : " << str << std::endl; - return xml2json(str); -} - -namespace emitters -{ - -/// \brief Generate string representation of json as an YAML document. -TOJSON_NODISCARD inline std::string toyaml(const json & j) -{ - YAML::Emitter e; - e << YAML::BeginDoc; - if (j.is_object()) { - e << YAML::BeginMap; - detail::toyaml(j, e); - e << YAML::EndMap; - } else if (j.is_array()) { - e << YAML::BeginSeq; - detail::toyaml(j, e); - e << YAML::EndSeq; - } - e << YAML::EndDoc; - return e.c_str(); -} - -/// \brief Generate string representation of json as an XML document. -/// \param j Json object to convert, must have a single root -/// \throws std::runtime_error if the object have more than one root -TOJSON_NODISCARD inline std::string toxml(const json & j) -{ - rapidxml::xml_document<> doc; - auto * decl = doc.allocate_node(rapidxml::node_declaration); - decl->append_attribute(doc.allocate_attribute("version", "1.0")); - decl->append_attribute(doc.allocate_attribute("encoding", "utf-8")); - doc.append_node(decl); - - if (j.is_object() && j.size() == 1) { - auto * root = - doc.allocate_node(rapidxml::node_element, doc.allocate_string(j.begin().key().data())); - detail::toxml(j.begin().value(), doc, root); - doc.append_node(root); - } else { - throw std::runtime_error("json must have a single root node"); - } - - std::string xml_as_string; - rapidxml::print(std::back_inserter(xml_as_string), doc); - return xml_as_string; -} - -} // namespace emitters -} // namespace tojson - -#endif //OPENSCENARIO_PREPROCESSOR_TOJSON_HPP diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 1dc7d08263e..8bf07e8079b 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #include #include -#include -#include -#include #include #include #include @@ -48,11 +47,7 @@ auto T4V2::deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vec base_scenario_ofs << split.second; base_scenario_ofs.close(); - auto openscenario_json = - tojson::loadyaml("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); - auto openscenario_xml = tojson::emitters::toxml(openscenario_json); - pugi::xml_document openscenario_doc; - openscenario_doc.load_string(openscenario_xml.c_str()); + auto openscenario_doc = loadScenarioFile("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); std::vector derived_scenarios; if (not split.first.empty()) { @@ -211,4 +206,45 @@ auto T4V2::deriveScenarioWithScenarioModifiers( return derived_scripts; } + +template +void convertYAMLtoXML(const YAML::Node & yaml, XMLClass & xml) +{ + switch (yaml.Type()) { + case YAML::NodeType::Scalar: + break; + case YAML::NodeType::Sequence: + break; + case YAML::NodeType::Map: + for (const auto & element : yaml) { + auto key = element.first.as(); + if (element.second.IsScalar()) { + xml.append_attribute(key.c_str()).set_value(element.second.as().c_str()); + } else if (element.second.IsSequence()) { + for (const auto & sequence_element : element.second) { + auto child = xml.append_child(key.c_str()); + convertYAMLtoXML(sequence_element, child); + } + } else { + auto child = xml.append_child(key.c_str()); + convertYAMLtoXML(element.second, child); + } + } + break; + case YAML::NodeType::Null: + break; + default: + throw std::runtime_error("Unknown YAML node type"); + } +} + +auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document +{ + std::cout << "loading scenario file: " << path << std::endl; + YAML::Node scenario_yaml = YAML::LoadFile(path.string()); + pugi::xml_document doc; + std::cout << "convert yaml to xml" << std::endl; + convertYAMLtoXML(scenario_yaml, doc); + return doc; +} } // namespace openscenario_preprocessor From c34632535ba9050b0a5722e342c62af973902d8d Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 1 May 2023 16:59:02 +0900 Subject: [PATCH 22/52] refactor(openscenario_preprocessor) --- .../src/openscenario_preprocessor.cpp | 42 ++++++++----------- .../src/openscenario_preprocessor_command.cpp | 19 --------- 2 files changed, 17 insertions(+), 44 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index 0a86debefb3..a8f63f853bd 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include namespace openscenario_preprocessor @@ -44,7 +43,6 @@ void Preprocessor::generateDerivedScenarioFromDistribution( openscenario_interpreter::ParameterDistribution & distribution, const boost::filesystem::path & path, ScenarioFormat output_format) { - // std::cout << "generateDerivedScenarioFromDistribution" << std::endl; for (const auto & parameter_list : distribution | boost::adaptors::indexed()) { pugi::xml_document derived_script; @@ -73,31 +71,25 @@ void Preprocessor::generateDerivedScenarioFromDistribution( output_directory / (path.stem().string() + "." + std::to_string(parameter_list.index()) + path.extension().string()); - try { - auto derived_scenario_path = [&]() { - if (output_format == ScenarioFormat::t4v2) { - YAML::Emitter yaml_emitter; - convertXMLtoYAML(derived_script, yaml_emitter); + auto derived_scenario_path = [&]() { + if (output_format == ScenarioFormat::t4v2) { + YAML::Emitter yaml_emitter; + convertXMLtoYAML(derived_script, yaml_emitter); - const auto derived_scenario_path_t4v2 = - output_directory / - (path.stem().string() + "." + std::to_string(parameter_list.index()) + ".yaml"); + const auto derived_scenario_path_t4v2 = + output_directory / + (path.stem().string() + "." + std::to_string(parameter_list.index()) + ".yaml"); - std::ofstream derived_scenario_yaml{derived_scenario_path_t4v2}; - derived_scenario_yaml << yaml_emitter.c_str(); - derived_scenario_yaml.close(); - return derived_scenario_path_t4v2; - } else { - derived_script.save_file(derived_scenario_path_xosc.c_str()); - return derived_scenario_path_xosc; - } - }(); - preprocessed_scenarios.emplace(derived_scenario_path); - } catch (rapidxml::parse_error & ex) { - std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() << ", " - << ex.where() << std::endl; - // return 1; - } + std::ofstream derived_scenario_yaml{derived_scenario_path_t4v2}; + derived_scenario_yaml << yaml_emitter.c_str(); + derived_scenario_yaml.close(); + return derived_scenario_path_t4v2; + } else { + derived_script.save_file(derived_scenario_path_xosc.c_str()); + return derived_scenario_path_xosc; + } + }(); + preprocessed_scenarios.emplace(derived_scenario_path); } } diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 2341a0b2cd9..14b4fa2a1ea 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -69,7 +69,6 @@ int main(const int argc, char const * const * const argv) options_description description("openscenario_preprocessor_command"); - // -o -p param.json -s scenario.yaml description.add_options()( "output-directory,o", value()->default_value("/tmp/openscenario_preprocessor/derived"), @@ -107,7 +106,6 @@ int main(const int argc, char const * const * const argv) boost::filesystem::create_directories(t4v2_output_directory); } for (const auto & xosc_string_scenario : xosc_string_scenarios | boost::adaptors::indexed()) { - // std::cout << xosc_string_scenario.value() << std::endl; boost::filesystem::path xosc_scenario_path = t4v2_output_directory / (std::to_string(xosc_string_scenario.index()) + ".xosc"); std::ofstream ofs(xosc_scenario_path.c_str()); @@ -115,7 +113,6 @@ int main(const int argc, char const * const * const argv) ofs.close(); xosc_scenario_paths.push_back(xosc_scenario_path); } - // std::cout << "finish writing xosc scenarios from t4v2 format" << std::endl; } else { xosc_scenario_paths.push_back(scenario_path); } @@ -131,24 +128,8 @@ int main(const int argc, char const * const * const argv) openscenario_preprocessor::Preprocessor preprocessor(output_directory_option); - // try { preprocessor.preprocessScenario( "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc", format_option); - // }catch (rapidxml::parse_error & ex){ - // std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() - // << ", " << ex.where() << std::endl; - //// return 1; - // } - // } catch (std::runtime_error & ex) { - // std::cerr << "[Error] something went wrong during deriving scenario : " << ex.what() - // << std::endl; - // return 1; - // }catch (std::exception & ex) { - // std::cerr << "[Error] unknown error occured during deriving scenario : " << ex.what() - // << std::endl; - // std::cerr << "Please contact developers of this software." << std::endl; - // return 1; - // } } } else { // TODO From dc64facc95e84ebc439adc85f6f511b01d98067c Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 1 May 2023 17:00:06 +0900 Subject: [PATCH 23/52] chore(openscenario_preprocessor): fix build error --- .../openscenario_preprocessor/src/openscenario_preprocessor.cpp | 1 - .../src/openscenario_preprocessor_command.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index a8f63f853bd..cb59a1eb7ee 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include namespace openscenario_preprocessor diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 14b4fa2a1ea..c7d3bac3cb1 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -17,7 +17,6 @@ #include #include #include -#include const std::string_view template_scenario = R"###( From 69f035cfc362894f746ab6da1ea51e28da660641 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 10 May 2023 16:40:43 +0900 Subject: [PATCH 24/52] feat(openscenario_preprocessor): add skip-full-derivation option to command --- .../openscenario_preprocessor.hpp | 5 ++ .../openscenario_preprocessor/t4v2.hpp | 7 ++- .../src/openscenario_preprocessor_command.cpp | 41 ++++++++++++++- .../openscenario_preprocessor/src/t4v2.cpp | 50 ++++++++++++------- 4 files changed, 80 insertions(+), 23 deletions(-) diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp index ef09186cb01..61060d5e9ac 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp @@ -69,6 +69,11 @@ class Preprocessor void convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & emitter); + const std::queue & getPreprocessedScenarios() const + { + return preprocessed_scenarios; + } + protected: std::queue preprocessed_scenarios; diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp index 2f8f0396eb4..b237db26dd2 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp @@ -24,7 +24,9 @@ namespace openscenario_preprocessor class T4V2 { public: - auto deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vector; + auto deriveToXoscStringScenarios( + boost::filesystem::path modifiers_path, boost::filesystem::path scenario_path = {}) + -> std::vector; auto loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document; @@ -36,7 +38,8 @@ class T4V2 const openscenario_interpreter::ParameterDistribution & scenario_modifier_distribution) -> std::vector; - std::pair splitScenarioModifiers(std::string scenario); + std::pair splitScenarioModifiers( + boost::filesystem::path scenario_path); }; } // namespace openscenario_preprocessor #endif //OPENSCENARIO_PREPROCESSOR_T4V2_HPP diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index c7d3bac3cb1..d363940f4bd 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -75,7 +75,8 @@ int main(const int argc, char const * const * const argv) "format,f", value()->multitoken(), "output scenario format (t4v2 / xosc)")( "parameters,p", value()->default_value("null"), "parameters in json format")( - "scenario,s", value(), "path of scenario file")("help,H", "help"); + "scenario,s", value(), "path of scenario file")("skip-full-derivation", "")( + "help,H", "help"); variables_map vm; store(parse_command_line(argc, argv, description), vm); @@ -85,6 +86,7 @@ int main(const int argc, char const * const * const argv) auto format_option = vm["format"].as(); auto parameters_option = boost::filesystem::path(vm["parameters"].as()); auto scenario_option = boost::filesystem::path(vm["scenario"].as()); + bool skip_full_derivation_option = (vm.count("skip-full-derivation") > 0); auto scenario_path = boost::filesystem::path(scenario_option); @@ -95,10 +97,22 @@ int main(const int argc, char const * const * const argv) std::vector xosc_scenario_paths; + boost::filesystem::path scenario_modifiers_path{}; + // preprocess t4v2 format and convert to xosc scenarios if (scenario_path.extension() == ".yaml" or scenario_path.extension() == ".yml") { openscenario_preprocessor::T4V2 t4v2; - auto xosc_string_scenarios = t4v2.deriveToXoscStringScenarios(scenario_path); + + auto [modifiers_path, base_scenario_path] = t4v2.splitScenarioModifiers(scenario_path); + scenario_modifiers_path = modifiers_path; + + auto xosc_string_scenarios = [&]() { + if (skip_full_derivation_option) { + return t4v2.deriveToXoscStringScenarios(base_scenario_path); + } else { + return t4v2.deriveToXoscStringScenarios(base_scenario_path, modifiers_path); + } + }(); boost::filesystem::path t4v2_output_directory = "/tmp/openscenario_preprocessor/t4v2_derived"; if (not boost::filesystem::exists(t4v2_output_directory)) { @@ -129,6 +143,29 @@ int main(const int argc, char const * const * const argv) preprocessor.preprocessScenario( "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc", format_option); + + // merge senario modifiers if skip_full_derivation_option is ON + if ( + format_option == openscenario_preprocessor::ScenarioFormat::t4v2 && + skip_full_derivation_option) { + auto derived_scenario_paths = preprocessor.getPreprocessedScenarios(); + while (not derived_scenario_paths.empty()) { + std::ifstream input_scenario_file{derived_scenario_paths.front().string()}; + std::stringstream scenario_ss; + scenario_ss << input_scenario_file.rdbuf(); + input_scenario_file.close(); + + std::ofstream scenario_file{derived_scenario_paths.front().string()}; + std::ifstream modifiers_file{scenario_modifiers_path.string()}; + scenario_file << modifiers_file.rdbuf(); + modifiers_file.close(); + + scenario_file << scenario_ss.str(); + scenario_file.close(); + + derived_scenario_paths.pop(); + } + } } } else { // TODO diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 8bf07e8079b..fe1b9d2d943 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -34,25 +34,20 @@ const std::string_view scenario_modifiers_distribution_base = R"###( )###"; -auto T4V2::deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vector +auto T4V2::deriveToXoscStringScenarios( + boost::filesystem::path scenario_path, boost::filesystem::path modifiers_path) + -> std::vector { - std::fstream file{path}; - std::stringstream scenario; - scenario << file.rdbuf(); - file.close(); - - auto split = splitScenarioModifiers(scenario.str()); - - std::ofstream base_scenario_ofs("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); - base_scenario_ofs << split.second; - base_scenario_ofs.close(); - - auto openscenario_doc = loadScenarioFile("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); + auto openscenario_doc = loadScenarioFile(scenario_path.string()); std::vector derived_scenarios; - if (not split.first.empty()) { + if (not modifiers_path.empty()) { + std::fstream file{modifiers_path.string()}; + std::stringstream modifiers_ss; + modifiers_ss << file.rdbuf(); + file.close(); auto parameter_value_distribution = - generateParameterValueDistributionFromScenarioModifiers(split.first); + generateParameterValueDistributionFromScenarioModifiers(modifiers_ss.str()); boost::filesystem::path parameter_value_distribution_path = "/tmp/openscenario_preprocessor/scenario_modifiers_distribution.xosc"; parameter_value_distribution.save_file(parameter_value_distribution_path.string().c_str()); @@ -83,14 +78,31 @@ auto T4V2::deriveToXoscStringScenarios(boost::filesystem::path path) -> std::vec return derived_scenarios; } -std::pair T4V2::splitScenarioModifiers(std::string scenario) +std::pair T4V2::splitScenarioModifiers( + boost::filesystem::path scenario_path) { + std::fstream file{scenario_path.string()}; + std::stringstream scenario_ss; + scenario_ss << file.rdbuf(); + file.close(); + + auto scenario_string = scenario_ss.str(); + std::regex re("(.*\n|^)(OpenSCENARIO.*)"); std::smatch match; - if (std::regex_search(scenario, match, re)) { - return std::make_pair( - scenario.substr(0, match.position(2)), scenario.substr(match.position(2))); + if (std::regex_search(scenario_string, match, re)) { + std::ofstream modifiers_ofs("/tmp/openscenario_preprocessor/t4v2_modifiers.yaml"); + modifiers_ofs << scenario_string.substr(0, match.position(2)); + modifiers_ofs.close(); + + std::ofstream base_scenario_ofs("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); + base_scenario_ofs << scenario_string.substr(match.position(2)); + base_scenario_ofs.close(); + + return std::make_pair( + "/tmp/openscenario_preprocessor/t4v2_modifiers.yaml", + "/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); } else { throw std::runtime_error( "No OpenSCENARIO element found in TIER IV 2.0 Format Scenario. Please check your " From f2d631ca92c9279c870b6702531f145a0f44b100 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 22 May 2023 13:35:19 +0900 Subject: [PATCH 25/52] feat(openscenario_preprocessor): implement xosc content order normalization with python --- .../openscenario_preprocessor/t4v2.hpp | 12 +-- .../openscenario_preprocessor/src/t4v2.cpp | 102 +++++++++++++++++- 2 files changed, 105 insertions(+), 9 deletions(-) diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp index b237db26dd2..859a540987c 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/t4v2.hpp @@ -24,19 +24,19 @@ namespace openscenario_preprocessor class T4V2 { public: + auto deriveScenarioWithScenarioModifiers( + const pugi::xml_document & base_scenario_doc, + const openscenario_interpreter::ParameterDistribution & scenario_modifier_distribution) + -> std::vector; + auto deriveToXoscStringScenarios( boost::filesystem::path modifiers_path, boost::filesystem::path scenario_path = {}) -> std::vector; - auto loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document; - auto generateParameterValueDistributionFromScenarioModifiers(std::string scenario_modifiers_str) -> pugi::xml_document; - auto deriveScenarioWithScenarioModifiers( - const pugi::xml_document & base_scenario_doc, - const openscenario_interpreter::ParameterDistribution & scenario_modifier_distribution) - -> std::vector; + auto loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document; std::pair splitScenarioModifiers( boost::filesystem::path scenario_path); diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index fe1b9d2d943..bc4a51851ab 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -34,6 +34,92 @@ const std::string_view scenario_modifiers_distribution_base = R"###( )###"; +const std::string_view load_yaml_to_xosc_with_encode_python_script = R"###( +import xmlschema +import yaml +import os +import sys +import xml.etree.ElementTree as ET + + +def from_yaml(keyword, node): + + if isinstance(node, dict): + # + # ???: { ... } + # + result = {} + + for tag, value in node.items(): + + if isinstance(value, list) and len(value) == 0: + # + # Tag: [] + # + # => REMOVE + # + continue + + if str.islower(tag[0]): + # + # tag: { ... } + # + # => @tag: { ... } + # + result["@" + tag] = str(value) + else: + # + # Tag: { ... } + # + # => NO CHANGES + # + result[tag] = from_yaml(tag, value) + + return result + + elif isinstance(node, list): + # + # ???: [ ... ] + # + result = [] + + for index, item in enumerate(node): + result.append(from_yaml(keyword, item)) + + return result + + elif isinstance(node, str): + return node + + else: + return None + + +if __name__ == "__main__": + # parse arguments here (input yaml path, output xosc path) + input_yaml_path = sys.argv[1] + output_xosc_path = sys.argv[2] + + xsd = open("/tmp/OpenSCENARIO-1.2.xsd") + schema = xmlschema.XMLSchema(xsd) + + if os.path.exists(input_yaml_path): + with open(input_yaml_path, "r") as file: + openscenario_yaml = from_yaml("OpenSCENARIO", yaml.safe_load(file)) + openscenario_yaml.pop("ScenarioModifiers", None) + xosc, errors = schema.encode( + openscenario_yaml, + indent=2, + preserve_root=True, + unordered=True, # Reorder elements + # The "strict" mode is too strict than we would like. + validation="lax", + ) + xosc = xmlschema.XMLResource(xosc).tostring().replace("True", "true").replace("False", "false") + with open(output_xosc_path, "w") as file: + file.write(xosc) +)###"; + auto T4V2::deriveToXoscStringScenarios( boost::filesystem::path scenario_path, boost::filesystem::path modifiers_path) -> std::vector @@ -253,10 +339,20 @@ void convertYAMLtoXML(const YAML::Node & yaml, XMLClass & xml) auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document { std::cout << "loading scenario file: " << path << std::endl; - YAML::Node scenario_yaml = YAML::LoadFile(path.string()); + std::string script_path = "/tmp/openscenario_preprocessor/load_yaml_to_xosc_with_encode.py"; + std::ofstream ofs(script_path); + ofs << load_yaml_to_xosc_with_encode_python_script; + ofs.close(); + + std::stringstream command_ss; + command_ss << "python3 " << script_path << " " << path.string() << " /tmp/openscenario_preprocessor/normalized.xosc"; + std::cout << command_ss.str() << std::endl; + + if(system(command_ss.str().c_str()) != 0){ + throw std::runtime_error("failed to execute python script : %s", command_ss.str()); + } pugi::xml_document doc; - std::cout << "convert yaml to xml" << std::endl; - convertYAMLtoXML(scenario_yaml, doc); + doc.load_file("/tmp/openscenario_preprocessor/normalized.xosc"); return doc; } } // namespace openscenario_preprocessor From f9bbb904402e89db678338bc42c1803922490465 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 22 May 2023 13:36:29 +0900 Subject: [PATCH 26/52] chore(openscenario_preprocessor): apply linter --- openscenario/openscenario_preprocessor/src/t4v2.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index bc4a51851ab..890f6f298f2 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -345,10 +345,11 @@ auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document ofs.close(); std::stringstream command_ss; - command_ss << "python3 " << script_path << " " << path.string() << " /tmp/openscenario_preprocessor/normalized.xosc"; + command_ss << "python3 " << script_path << " " << path.string() + << " /tmp/openscenario_preprocessor/normalized.xosc"; std::cout << command_ss.str() << std::endl; - if(system(command_ss.str().c_str()) != 0){ + if (system(command_ss.str().c_str()) != 0) { throw std::runtime_error("failed to execute python script : %s", command_ss.str()); } pugi::xml_document doc; From c6c75e16347d77824b5b4e7e628637892573158e Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 22 May 2023 13:49:53 +0900 Subject: [PATCH 27/52] fix(openscenario_preprocessor): fix compile error --- openscenario/openscenario_preprocessor/src/t4v2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 890f6f298f2..442f08b219a 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -350,7 +350,7 @@ auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document std::cout << command_ss.str() << std::endl; if (system(command_ss.str().c_str()) != 0) { - throw std::runtime_error("failed to execute python script : %s", command_ss.str()); + throw std::runtime_error("failed to execute python script : " + command_ss.str()); } pugi::xml_document doc; doc.load_file("/tmp/openscenario_preprocessor/normalized.xosc"); From 365abb50d93888ce585914c8ba89ead69ac581a2 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 22 May 2023 18:46:54 +0900 Subject: [PATCH 28/52] fix(openscenario_preprocessor): load schema --- .../src/openscenario_preprocessor_command.cpp | 13 +++++++++++-- openscenario/openscenario_preprocessor/src/t4v2.cpp | 5 ++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index d363940f4bd..2f4f92c9b10 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -17,6 +17,7 @@ #include #include #include +#include const std::string_view template_scenario = R"###( @@ -62,7 +63,7 @@ auto create_parameter_value_distribution_from_json( return script; } -int main(const int argc, char const * const * const argv) +int main(const int argc, char const * const * const argv) try { using namespace boost::program_options; @@ -99,6 +100,11 @@ int main(const int argc, char const * const * const argv) boost::filesystem::path scenario_modifiers_path{}; + // setup xsd file + auto file = std::ofstream("/tmp/openscenario_preprocessor/schema.xsd", std::ios::trunc); + file << openscenario_validator::schema; + file.close(); + // preprocess t4v2 format and convert to xosc scenarios if (scenario_path.extension() == ".yaml" or scenario_path.extension() == ".yml") { openscenario_preprocessor::T4V2 t4v2; @@ -168,7 +174,10 @@ int main(const int argc, char const * const * const argv) } } } else { - // TODO + throw std::runtime_error("parameters option is required"); } return 0; +} catch (std::exception & e) { + std::cerr << "Caught an exception : " << e.what() << std::endl; + return 1; } diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 442f08b219a..ec715658d9a 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -100,7 +101,7 @@ if __name__ == "__main__": input_yaml_path = sys.argv[1] output_xosc_path = sys.argv[2] - xsd = open("/tmp/OpenSCENARIO-1.2.xsd") + xsd = open("/tmp/openscenario_preprocessor/schema.xsd") schema = xmlschema.XMLSchema(xsd) if os.path.exists(input_yaml_path): @@ -338,7 +339,6 @@ void convertYAMLtoXML(const YAML::Node & yaml, XMLClass & xml) auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document { - std::cout << "loading scenario file: " << path << std::endl; std::string script_path = "/tmp/openscenario_preprocessor/load_yaml_to_xosc_with_encode.py"; std::ofstream ofs(script_path); ofs << load_yaml_to_xosc_with_encode_python_script; @@ -347,7 +347,6 @@ auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document std::stringstream command_ss; command_ss << "python3 " << script_path << " " << path.string() << " /tmp/openscenario_preprocessor/normalized.xosc"; - std::cout << command_ss.str() << std::endl; if (system(command_ss.str().c_str()) != 0) { throw std::runtime_error("failed to execute python script : " + command_ss.str()); From c3424154188f325bca8f514311784599dc5474e4 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 24 May 2023 10:24:56 +0900 Subject: [PATCH 29/52] chore(openscenario_preprocessor): apply linter --- .../src/openscenario_preprocessor_command.cpp | 4 ++-- openscenario/openscenario_preprocessor/src/t4v2.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 2f4f92c9b10..308afcd3b74 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -63,8 +63,8 @@ auto create_parameter_value_distribution_from_json( return script; } -int main(const int argc, char const * const * const argv) try -{ +int main(const int argc, char const * const * const argv) +try { using namespace boost::program_options; options_description description("openscenario_preprocessor_command"); diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index ec715658d9a..c43b07dffce 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -17,9 +17,9 @@ #include #include #include -#include #include #include +#include #include #include From 191cc0855d89952b9a266fa7bfed2a96eab3c350 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 24 May 2023 10:53:25 +0900 Subject: [PATCH 30/52] chore(openscenario_preprocessor): fix cspell errors --- openscenario/openscenario_interpreter/CMakeLists.txt | 2 +- openscenario/openscenario_preprocessor/CMakeLists.txt | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/openscenario/openscenario_interpreter/CMakeLists.txt b/openscenario/openscenario_interpreter/CMakeLists.txt index df29d823e22..bf8abe4afea 100644 --- a/openscenario/openscenario_interpreter/CMakeLists.txt +++ b/openscenario/openscenario_interpreter/CMakeLists.txt @@ -107,7 +107,7 @@ add_library(parameter_value_distribution_without_ros STATIC src/scope.cpp) add_definitions("-DBOOST_ALLOW_DEPRECATED_HEADERS") -# cspell: ignore DWITHOUT +# cspell: ignore DPARAMETER target_compile_definitions(parameter_value_distribution_without_ros PUBLIC -DPARAMETER_VALUE_DISTRIBUTION_ONLY) target_include_directories(parameter_value_distribution_without_ros PUBLIC include) target_include_directories(parameter_value_distribution_without_ros PUBLIC ${scenario_simulator_exception_INCLUDE_DIRS}) diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 22f05c9ea07..9a092ab4cf8 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -24,13 +24,14 @@ ament_auto_add_executable(${PROJECT_NAME}_node target_link_libraries(${PROJECT_NAME}_node XercesC::XercesC) target_link_libraries(${PROJECT_NAME}_node yaml-cpp) -find_library(parameter_value_distribution_LIBRARYIES parameter_value_distribution_without_ros) +find_library(parameter_value_distribution_LIBRARIES parameter_value_distribution_without_ros) add_executable(${PROJECT_NAME}_command src/t4v2.cpp src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_command.cpp) +# cspell: ignore DPARAMETER target_compile_definitions(${PROJECT_NAME}_command PUBLIC -DPARAMETER_VALUE_DISTRIBUTION_ONLY) target_include_directories(${PROJECT_NAME}_command PUBLIC include) @@ -45,7 +46,7 @@ target_link_libraries(${PROJECT_NAME}_command XercesC::XercesC) target_link_libraries(${PROJECT_NAME}_command pugixml) target_link_libraries(${PROJECT_NAME}_command yaml-cpp) -target_link_libraries(${PROJECT_NAME}_command ${parameter_value_distribution_LIBRARYIES}) +target_link_libraries(${PROJECT_NAME}_command ${parameter_value_distribution_LIBRARIES}) target_link_libraries(${PROJECT_NAME}_command ${openscenario_validator_LIBRARIES}) install(TARGETS ${PROJECT_NAME}_node ${PROJECT_NAME}_command From 7b2eae369e8e9bc8752f26c5de564bd2ef01b584 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 24 May 2023 10:53:47 +0900 Subject: [PATCH 31/52] chore(openscenario_preprocessor): delete unused dependencies --- openscenario/openscenario_preprocessor/package.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/openscenario/openscenario_preprocessor/package.xml b/openscenario/openscenario_preprocessor/package.xml index 620c1b451e1..ebf7b493643 100644 --- a/openscenario/openscenario_preprocessor/package.xml +++ b/openscenario/openscenario_preprocessor/package.xml @@ -16,7 +16,6 @@ xerces pugixml-dev yaml-cpp - librapidxml-dev nlohmann-json-dev ament_cmake_clang_format From 5138ea3a4e28c503a7fb70223112929aa5fb287c Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 24 May 2023 10:54:00 +0900 Subject: [PATCH 32/52] chore(openscenario_preprocessor): delete unused files --- .../template_distributions.hpp | 27 --------------- .../openscenario_preprocessor/src/t4v2.cpp | 1 - .../src/template_distribution_range.cpp | 33 ------------------ .../src/template_value_set_distribution.cpp | 34 ------------------- 4 files changed, 95 deletions(-) delete mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp delete mode 100644 openscenario/openscenario_preprocessor/src/template_distribution_range.cpp delete mode 100644 openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp deleted file mode 100644 index 7baaf4d9c8e..00000000000 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/template_distributions.hpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2015 TIER IV, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPENSCENARIO_PREPROCESSOR_TEMPLATE_DISTRIBUTIONS_HPP -#define OPENSCENARIO_PREPROCESSOR_TEMPLATE_DISTRIBUTIONS_HPP - -#include - -namespace openscenario_preprocessor -{ -extern const std::string_view template_value_set_distribution; - -extern const std::string_view template_distribution_range; -} // namespace openscenario_preprocessor - -#endif //OPENSCENARIO_PREPROCESSOR_TEMPLATE_DISTRIBUTIONS_HPP diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index c43b07dffce..114f2b9d47f 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include diff --git a/openscenario/openscenario_preprocessor/src/template_distribution_range.cpp b/openscenario/openscenario_preprocessor/src/template_distribution_range.cpp deleted file mode 100644 index 10e7d8a3f8f..00000000000 --- a/openscenario/openscenario_preprocessor/src/template_distribution_range.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2015 TIER IV, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -namespace openscenario_preprocessor -{ -const std::string_view template_distribution_trange = R"###( - - - - - - - - - - - - -)###"; -} // namespace openscenario_preprocessor diff --git a/openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp b/openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp deleted file mode 100644 index 13fb9156c02..00000000000 --- a/openscenario/openscenario_preprocessor/src/template_value_set_distribution.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2015 TIER IV, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -namespace openscenario_preprocessor -{ -const std::string_view template_value_set_distribution = R"###( - - - - - - - - - - - - - -)###"; -} // namespace openscenario_preprocessor From 9c030a8ccecfabc4c4839029d87997a254b111ce Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 24 May 2023 10:56:09 +0900 Subject: [PATCH 33/52] chore(openscenario_preprocessor): fix cspell errors --- .../src/openscenario_preprocessor_command.cpp | 3 ++- openscenario/openscenario_preprocessor/src/t4v2.cpp | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 308afcd3b74..7eff5eebc82 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -69,6 +69,7 @@ try { options_description description("openscenario_preprocessor_command"); +// cspell: ignore multitoken description.add_options()( "output-directory,o", value()->default_value("/tmp/openscenario_preprocessor/derived"), @@ -150,7 +151,7 @@ try { preprocessor.preprocessScenario( "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc", format_option); - // merge senario modifiers if skip_full_derivation_option is ON + // merge scenario modifiers if skip_full_derivation_option is ON if ( format_option == openscenario_preprocessor::ScenarioFormat::t4v2 && skip_full_derivation_option) { diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 114f2b9d47f..038b9493685 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -24,6 +24,9 @@ namespace openscenario_preprocessor { + +// cspell: ignore isinstance etree + const std::string_view scenario_modifiers_distribution_base = R"###( From e548257c7fb0c5febd671cc5396e5433bddb17f2 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Thu, 25 May 2023 17:28:09 +0900 Subject: [PATCH 34/52] chore: apply linter --- .../src/openscenario_preprocessor_command.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 7eff5eebc82..ae768c56b0d 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -69,7 +69,7 @@ try { options_description description("openscenario_preprocessor_command"); -// cspell: ignore multitoken + // cspell: ignore multitoken description.add_options()( "output-directory,o", value()->default_value("/tmp/openscenario_preprocessor/derived"), From da5794cc88e96ed379b213eba3f6f2136099a48d Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 29 May 2023 15:13:47 +0900 Subject: [PATCH 35/52] convert json values to string to avoid error --- .../src/openscenario_preprocessor_command.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index ae768c56b0d..155f9dfe28d 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -140,8 +140,23 @@ try { // derive for given parameters if (parameters_option != "null") { for (auto scenario_path : xosc_scenario_paths) { - auto parameter_value_distribution = create_parameter_value_distribution_from_json( - scenario_path, nlohmann::json::parse(parameters_option.c_str())); + auto parameters_json = nlohmann::json::parse(parameters_option.c_str()); + // convert value to string, because current implementation is not support other types + for (auto & json_item : parameters_json.items()) { + if (not json_item.value().is_string()) { + try { + json_item.value() = boost::lexical_cast(json_item.value()); + } catch (std::exception & e) { + std::stringstream what; + what << "Cannot convert parameter value, type " << json_item.value().type_name() + << " to string : " << e.what(); + throw std::runtime_error(what.str()); + } + } + } + + auto parameter_value_distribution = + create_parameter_value_distribution_from_json(scenario_path, parameters_json); parameter_value_distribution.save_file( "/tmp/openscenario_preprocessor/parameter_value_distribution.xosc"); From 2a080fec902775ab770b805d3566315a9ad3edac Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Thu, 1 Jun 2023 18:23:20 +0900 Subject: [PATCH 36/52] chore(openscenario_preprocessor): write-out intermediate file for debug --- .../src/openscenario_preprocessor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index cb59a1eb7ee..ab5e8b5f193 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -82,6 +82,12 @@ void Preprocessor::generateDerivedScenarioFromDistribution( std::ofstream derived_scenario_yaml{derived_scenario_path_t4v2}; derived_scenario_yaml << yaml_emitter.c_str(); derived_scenario_yaml.close(); + + // NOTE: for debug, write out yaml to "/tmp/openscenario_preprocessor/preprocessed_scenario.yaml" + std::ofstream debug_yaml{"/tmp/openscenario_preprocessor/preprocessed_scenario.yaml"}; + debug_yaml << yaml_emitter.c_str(); + debug_yaml.close(); + return derived_scenario_path_t4v2; } else { derived_script.save_file(derived_scenario_path_xosc.c_str()); From 59fa833356ce5ff6b51ba7761b3b53c91b3e3769 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 2 Jun 2023 17:44:10 +0900 Subject: [PATCH 37/52] chore(openscenario_preprocessor): add debug message --- .../src/openscenario_preprocessor_command.cpp | 64 +++++++++++++++++-- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 155f9dfe28d..cf69c798a76 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -172,19 +172,71 @@ try { skip_full_derivation_option) { auto derived_scenario_paths = preprocessor.getPreprocessedScenarios(); while (not derived_scenario_paths.empty()) { - std::ifstream input_scenario_file{derived_scenario_paths.front().string()}; + + // 1. copy scenario content to stringstream + std::cout << "1. copy scenario content to stringstream" << std::endl; std::stringstream scenario_ss; + std::ifstream input_scenario_file{derived_scenario_paths.front().string()}; + if(not input_scenario_file){ + std::stringstream what; + what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); + throw std::runtime_error(what.str()); + }else{ + std::cout << "opened scenario file : " << derived_scenario_paths.front().string() << std::endl; + } scenario_ss << input_scenario_file.rdbuf(); + std::string scenario_str = scenario_ss.str(); input_scenario_file.close(); - std::ofstream scenario_file{derived_scenario_paths.front().string()}; + std::cout << "file closed" << std::endl; + std::cout << "-------------------------------------" << std::endl; + std::cout << "copied scenario content : " << scenario_str << std::endl; + std::cout << "-------------------------------------" << std::endl; + + // 2. write scenario modifiers to scenario file + std::cout << "2. write scenario modifiers to scenario file" << std::endl; + + std::stringstream merged_scenario_ss; + std::ifstream modifiers_file{scenario_modifiers_path.string()}; - scenario_file << modifiers_file.rdbuf(); - modifiers_file.close(); + if(not modifiers_file){ + std::stringstream what; + what << "Cannot open scenario modifiers file : " << scenario_modifiers_path.string(); + throw std::runtime_error(what.str()); + } + + std::cout << "append scenario modifiers to scenario file" << std::endl; + std::cout << "-------------------------------------" << std::endl; + std::cout << "scenario modifiers content : " << modifiers_file.rdbuf() << std::endl; + std::cout << "-------------------------------------" << std::endl; + merged_scenario_ss << modifiers_file.rdbuf() << std::endl; + + std::cout << "scenario modifiers file closed" << std::endl; + + std::cout << "-------------------------------------" << std::endl; + std::cout << "merged_scenario_ss : " << merged_scenario_ss.str() << std::endl; + std::cout << "-------------------------------------" << std::endl; + std::cout << std::endl; + std::cout << std::endl; + + merged_scenario_ss << scenario_str << std::endl; + std::cout << "scenario content to be written" << std::endl; + std::cout << "-------------------------------------" << std::endl; + std::cout << "written scenario content : " << merged_scenario_ss.str() << std::endl; + std::cout << "-------------------------------------" << std::endl; + +// std::ofstream scenario_file(derived_scenario_paths.front().string(), std::ios::trunc); +// if(not scenario_file){ +// std::stringstream what; +// what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); +// throw std::runtime_error(what.str()); +// } +// scenario_file << merged_scenario_ss.str(); +// scenario_file.close(); + std::cout << "scenario file closed" << std::endl; - scenario_file << scenario_ss.str(); - scenario_file.close(); + modifiers_file.close(); derived_scenario_paths.pop(); } } From 76ab42160601166213cfe9248cf83ac10303dff3 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 2 Jun 2023 18:47:25 +0900 Subject: [PATCH 38/52] chore(openscenario_preprocessor): load unordered yaml --- .../src/openscenario_preprocessor_command.cpp | 27 ++++++------ .../openscenario_preprocessor/src/t4v2.cpp | 41 ++++++++++++++----- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index cf69c798a76..aa5b4122616 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -172,17 +172,17 @@ try { skip_full_derivation_option) { auto derived_scenario_paths = preprocessor.getPreprocessedScenarios(); while (not derived_scenario_paths.empty()) { - // 1. copy scenario content to stringstream std::cout << "1. copy scenario content to stringstream" << std::endl; std::stringstream scenario_ss; std::ifstream input_scenario_file{derived_scenario_paths.front().string()}; - if(not input_scenario_file){ + if (not input_scenario_file) { std::stringstream what; what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); throw std::runtime_error(what.str()); - }else{ - std::cout << "opened scenario file : " << derived_scenario_paths.front().string() << std::endl; + } else { + std::cout << "opened scenario file : " << derived_scenario_paths.front().string() + << std::endl; } scenario_ss << input_scenario_file.rdbuf(); std::string scenario_str = scenario_ss.str(); @@ -199,7 +199,7 @@ try { std::stringstream merged_scenario_ss; std::ifstream modifiers_file{scenario_modifiers_path.string()}; - if(not modifiers_file){ + if (not modifiers_file) { std::stringstream what; what << "Cannot open scenario modifiers file : " << scenario_modifiers_path.string(); throw std::runtime_error(what.str()); @@ -225,17 +225,16 @@ try { std::cout << "written scenario content : " << merged_scenario_ss.str() << std::endl; std::cout << "-------------------------------------" << std::endl; -// std::ofstream scenario_file(derived_scenario_paths.front().string(), std::ios::trunc); -// if(not scenario_file){ -// std::stringstream what; -// what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); -// throw std::runtime_error(what.str()); -// } -// scenario_file << merged_scenario_ss.str(); -// scenario_file.close(); + // std::ofstream scenario_file(derived_scenario_paths.front().string(), std::ios::trunc); + // if(not scenario_file){ + // std::stringstream what; + // what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); + // throw std::runtime_error(what.str()); + // } + // scenario_file << merged_scenario_ss.str(); + // scenario_file.close(); std::cout << "scenario file closed" << std::endl; - modifiers_file.close(); derived_scenario_paths.pop(); } diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 038b9493685..e9f7339ba15 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -177,25 +177,44 @@ std::pair T4V2::splitScenarioM auto scenario_string = scenario_ss.str(); - std::regex re("(.*\n|^)(OpenSCENARIO.*)"); - std::smatch match; + ; + if (auto modifiers_pos = scenario_string.find("ScenarioModifiers"); + modifiers_pos != std::string::npos) { + auto openscenario_pos = scenario_string.find("OpenSCENARIO"); + + if (openscenario_pos == std::string::npos) { + throw std::runtime_error( + "No OpenSCENARIO element found in TIER IV 2.0 Format Scenario. Please check your " + "scenario."); + } - if (std::regex_search(scenario_string, match, re)) { std::ofstream modifiers_ofs("/tmp/openscenario_preprocessor/t4v2_modifiers.yaml"); - modifiers_ofs << scenario_string.substr(0, match.position(2)); - modifiers_ofs.close(); - std::ofstream base_scenario_ofs("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); - base_scenario_ofs << scenario_string.substr(match.position(2)); - base_scenario_ofs.close(); + if (openscenario_pos > modifiers_pos) { + std::regex re("(.*\n|^)(OpenSCENARIO.*)"); + std::smatch match; + std::regex_search(scenario_string, match, re); + modifiers_ofs << scenario_string.substr(0, match.position(2)); + base_scenario_ofs << scenario_string.substr(match.position(2)); + } else { + std::regex re("(.*\n|^)(ScenarioModifiers.*)"); + std::smatch match; + std::regex_search(scenario_string, match, re); + base_scenario_ofs << scenario_string.substr(0, match.position(2)); + modifiers_ofs << scenario_string.substr(match.position(2)); + } + modifiers_ofs.close(); + base_scenario_ofs.close(); return std::make_pair( "/tmp/openscenario_preprocessor/t4v2_modifiers.yaml", "/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); } else { - throw std::runtime_error( - "No OpenSCENARIO element found in TIER IV 2.0 Format Scenario. Please check your " - "scenario."); + std::ofstream base_scenario_ofs("/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); + base_scenario_ofs << scenario_string; + base_scenario_ofs.close(); + return std::make_pair( + "", "/tmp/openscenario_preprocessor/t4v2_openscenario.yaml"); } } From d7a03248913844cd9ab46dcbd72fc0301522eb21 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 5 Jun 2023 20:34:23 +0900 Subject: [PATCH 39/52] fix(openscenario_preprocessor): fix bug in coversion from xml to yaml --- .../src/openscenario_preprocessor.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp index ab5e8b5f193..234a8672e68 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor.cpp @@ -83,7 +83,8 @@ void Preprocessor::generateDerivedScenarioFromDistribution( derived_scenario_yaml << yaml_emitter.c_str(); derived_scenario_yaml.close(); - // NOTE: for debug, write out yaml to "/tmp/openscenario_preprocessor/preprocessed_scenario.yaml" + // NOTE: for debug, write out yaml to + // "/tmp/openscenario_preprocessor/preprocessed_scenario.yaml" std::ofstream debug_yaml{"/tmp/openscenario_preprocessor/preprocessed_scenario.yaml"}; debug_yaml << yaml_emitter.c_str(); debug_yaml.close(); @@ -108,7 +109,7 @@ void Preprocessor::convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & // the map of node name and {total count, used count} std::map> count; for (const auto & child : xml.children()) { - if (count.find(child.name()) != count.end()) { + if (count.find(child.name()) == count.end()) { count[child.name()] = std::make_pair(1, 0); } else { count[child.name()].first++; @@ -123,7 +124,9 @@ void Preprocessor::convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & emitter << YAML::Key << attr.name(); emitter << YAML::Value << YAML::SingleQuoted << attr.as_string(); } - // end map in child element part + if (xml.children().empty()) { + emitter << YAML::EndMap; + } } // iterate child elements @@ -131,21 +134,21 @@ void Preprocessor::convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & if (xml.attributes().empty()) { emitter << YAML::BeginMap; } - for (const auto & child : xml.children()) { - // count == 1, make single map in yaml + // total count == 1, make single map in yaml if (count[child.name()].first == 1) { emitter << YAML::Key << child.name(); emitter << YAML::Value; convertXMLtoYAML(child, emitter); } else { - // count > 1, make sequence in yaml + // total count > 1, make sequence in yaml if (count[child.name()].second == 0) { // first node - emitter << YAML::BeginSeq; emitter << YAML::Key << child.name(); emitter << YAML::Value; + emitter << YAML::BeginSeq; convertXMLtoYAML(child, emitter); + // count up used count count[child.name()].second++; } else if (count[child.name()].second == count[child.name()].first - 1) { // last node @@ -158,9 +161,6 @@ void Preprocessor::convertXMLtoYAML(const pugi::xml_node & xml, YAML::Emitter & } } } - } - - if (not xml.attributes().empty() or not xml.children().empty()) { emitter << YAML::EndMap; } } From b9536ded359265ae2489619fce229ce2d7fa184d Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 5 Jun 2023 20:34:52 +0900 Subject: [PATCH 40/52] refactor(openscenario_preprocessor) --- .../src/openscenario_preprocessor_command.cpp | 94 ++++++++----------- .../openscenario_preprocessor/src/t4v2.cpp | 60 ++++++------ 2 files changed, 70 insertions(+), 84 deletions(-) diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index aa5b4122616..0d6f582dc65 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -173,69 +173,55 @@ try { auto derived_scenario_paths = preprocessor.getPreprocessedScenarios(); while (not derived_scenario_paths.empty()) { // 1. copy scenario content to stringstream - std::cout << "1. copy scenario content to stringstream" << std::endl; std::stringstream scenario_ss; - std::ifstream input_scenario_file{derived_scenario_paths.front().string()}; - if (not input_scenario_file) { - std::stringstream what; - what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); - throw std::runtime_error(what.str()); - } else { - std::cout << "opened scenario file : " << derived_scenario_paths.front().string() - << std::endl; + { + std::ifstream input_scenario_file{derived_scenario_paths.front().string()}; + if (not input_scenario_file) { + std::stringstream what; + what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); + throw std::runtime_error(what.str()); + } else { + std::cout << "opened scenario file : " << derived_scenario_paths.front().string() + << std::endl; + } + scenario_ss << input_scenario_file.rdbuf(); + input_scenario_file.close(); } - scenario_ss << input_scenario_file.rdbuf(); - std::string scenario_str = scenario_ss.str(); - input_scenario_file.close(); - - std::cout << "file closed" << std::endl; - std::cout << "-------------------------------------" << std::endl; - std::cout << "copied scenario content : " << scenario_str << std::endl; - std::cout << "-------------------------------------" << std::endl; // 2. write scenario modifiers to scenario file std::cout << "2. write scenario modifiers to scenario file" << std::endl; - std::stringstream merged_scenario_ss; - std::ifstream modifiers_file{scenario_modifiers_path.string()}; - if (not modifiers_file) { - std::stringstream what; - what << "Cannot open scenario modifiers file : " << scenario_modifiers_path.string(); - throw std::runtime_error(what.str()); + // load scenario modifiers + { + std::ifstream modifiers_file{scenario_modifiers_path.string()}; + if (not modifiers_file) { + std::stringstream what; + what << "Cannot open scenario modifiers file : " << scenario_modifiers_path.string(); + throw std::runtime_error(what.str()); + } + + merged_scenario_ss << modifiers_file.rdbuf(); + modifiers_file.close(); + } + + // load scenario content + { + merged_scenario_ss << scenario_ss.str(); + } + + // write-out merged scenario + { + std::ofstream scenario_file(derived_scenario_paths.front().string()); + if (not scenario_file) { + std::stringstream what; + what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); + throw std::runtime_error(what.str()); + } + scenario_file << merged_scenario_ss.str(); + scenario_file.close(); } - std::cout << "append scenario modifiers to scenario file" << std::endl; - std::cout << "-------------------------------------" << std::endl; - std::cout << "scenario modifiers content : " << modifiers_file.rdbuf() << std::endl; - std::cout << "-------------------------------------" << std::endl; - merged_scenario_ss << modifiers_file.rdbuf() << std::endl; - - std::cout << "scenario modifiers file closed" << std::endl; - - std::cout << "-------------------------------------" << std::endl; - std::cout << "merged_scenario_ss : " << merged_scenario_ss.str() << std::endl; - std::cout << "-------------------------------------" << std::endl; - std::cout << std::endl; - std::cout << std::endl; - - merged_scenario_ss << scenario_str << std::endl; - std::cout << "scenario content to be written" << std::endl; - std::cout << "-------------------------------------" << std::endl; - std::cout << "written scenario content : " << merged_scenario_ss.str() << std::endl; - std::cout << "-------------------------------------" << std::endl; - - // std::ofstream scenario_file(derived_scenario_paths.front().string(), std::ios::trunc); - // if(not scenario_file){ - // std::stringstream what; - // what << "Cannot open scenario file : " << derived_scenario_paths.front().string(); - // throw std::runtime_error(what.str()); - // } - // scenario_file << merged_scenario_ss.str(); - // scenario_file.close(); - std::cout << "scenario file closed" << std::endl; - - modifiers_file.close(); derived_scenario_paths.pop(); } } diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index e9f7339ba15..4b5dd4f0607 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -327,36 +327,36 @@ auto T4V2::deriveScenarioWithScenarioModifiers( return derived_scripts; } -template -void convertYAMLtoXML(const YAML::Node & yaml, XMLClass & xml) -{ - switch (yaml.Type()) { - case YAML::NodeType::Scalar: - break; - case YAML::NodeType::Sequence: - break; - case YAML::NodeType::Map: - for (const auto & element : yaml) { - auto key = element.first.as(); - if (element.second.IsScalar()) { - xml.append_attribute(key.c_str()).set_value(element.second.as().c_str()); - } else if (element.second.IsSequence()) { - for (const auto & sequence_element : element.second) { - auto child = xml.append_child(key.c_str()); - convertYAMLtoXML(sequence_element, child); - } - } else { - auto child = xml.append_child(key.c_str()); - convertYAMLtoXML(element.second, child); - } - } - break; - case YAML::NodeType::Null: - break; - default: - throw std::runtime_error("Unknown YAML node type"); - } -} +//template +//void convertYAMLtoXML(const YAML::Node & yaml, XMLClass & xml) +//{ +// switch (yaml.Type()) { +// case YAML::NodeType::Scalar: +// break; +// case YAML::NodeType::Sequence: +// break; +// case YAML::NodeType::Map: +// for (const auto & element : yaml) { +// auto key = element.first.as(); +// if (element.second.IsScalar()) { +// xml.append_attribute(key.c_str()).set_value(element.second.as().c_str()); +// } else if (element.second.IsSequence()) { +// for (const auto & sequence_element : element.second) { +// auto child = xml.append_child(key.c_str()); +// convertYAMLtoXML(sequence_element, child); +// } +// } else { +// auto child = xml.append_child(key.c_str()); +// convertYAMLtoXML(element.second, child); +// } +// } +// break; +// case YAML::NodeType::Null: +// break; +// default: +// throw std::runtime_error("Unknown YAML node type"); +// } +//} auto T4V2::loadScenarioFile(boost::filesystem::path path) -> pugi::xml_document { From a597cc67d7d4248b8890812c1a6e28833f86d243 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Tue, 6 Jun 2023 16:35:43 +0900 Subject: [PATCH 41/52] chore(openscenario_preprocessor): (wip) add test --- .../openscenario_preprocessor/CMakeLists.txt | 1 + .../openscenario_preprocessor/package.xml | 1 + .../test/test_main.cpp | 62 +++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 openscenario/openscenario_preprocessor/test/test_main.cpp diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 9a092ab4cf8..d86b7d949f4 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -56,6 +56,7 @@ install(TARGETS ${PROJECT_NAME}_node ${PROJECT_NAME}_command if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() +# ament_auto_add_gtest(${PROJECT_NAME}_test DIRECTORY "test/") endif() ament_auto_package() diff --git a/openscenario/openscenario_preprocessor/package.xml b/openscenario/openscenario_preprocessor/package.xml index ebf7b493643..9ef1d61bb98 100644 --- a/openscenario/openscenario_preprocessor/package.xml +++ b/openscenario/openscenario_preprocessor/package.xml @@ -24,6 +24,7 @@ ament_cmake_pep257 ament_cmake_xmllint ament_lint_auto + ament_cmake_gtest ament_cmake diff --git a/openscenario/openscenario_preprocessor/test/test_main.cpp b/openscenario/openscenario_preprocessor/test/test_main.cpp new file mode 100644 index 00000000000..26afaf4e5bd --- /dev/null +++ b/openscenario/openscenario_preprocessor/test/test_main.cpp @@ -0,0 +1,62 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" + +#include + +auto dollar(const std::string & command) -> std::string +{ + std::array buffer; + + std::string result; + + std::unique_ptr pipe{::popen(command.c_str(), "r"), pclose}; + + if (not pipe) { + throw std::system_error(errno, std::system_category()); + } else { + while (std::fgets(buffer.data(), buffer.size(), pipe.get())) { + result += buffer.data(); + } + return result; + } +} + +TEST(getYawFromQuaternion, returnValue) +{ + std::stringstream preprocessor_command; + preprocessor_command << "ros2 run openscenario_preprocessor openscenario_preprocessor_command " + "-s /home/hans/Documents/sample.yaml " + "--parameters '{\"random_offset\": true}' " + "-f t4v2 -o /home/hans/Documents/cmd " + "--skip-full-derivation"; + dollar(preprocessor_command.str()); + + std::stringstream diff_command; + diff_command << " diff " + << "-bB " + << "/tmp/scenario_test_runner/0.0/0.0_0.xosc" + << " /tmp/openscenario_preprocessor/normalized.xosc" + << " | sed '/^[^>].*/d' | wc -l"; + std::string diff_result = dollar(diff_command.str()); + + EXPECT_EQ(1, boost::lexical_cast(diff_result)); +} + +auto main(int argc, char ** argv) -> int +{ + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 17a316cc746a17a345fda48491f486c26b1640c5 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 7 Jun 2023 16:38:30 +0900 Subject: [PATCH 42/52] feat(preprocessor): add some test --- .../openscenario_preprocessor/CMakeLists.txt | 2 +- .../openscenario_preprocessor/package.xml | 4 +- .../test/test_main.cpp | 62 ------------------- .../test/test_main.py | 50 +++++++++++++++ 4 files changed, 54 insertions(+), 64 deletions(-) delete mode 100644 openscenario/openscenario_preprocessor/test/test_main.cpp create mode 100644 openscenario/openscenario_preprocessor/test/test_main.py diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index d86b7d949f4..49f3408767f 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -56,7 +56,7 @@ install(TARGETS ${PROJECT_NAME}_node ${PROJECT_NAME}_command if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() -# ament_auto_add_gtest(${PROJECT_NAME}_test DIRECTORY "test/") + ament_add_pytest_test(test_${PROJECT_NAME} "test/test_main.py") endif() ament_auto_package() diff --git a/openscenario/openscenario_preprocessor/package.xml b/openscenario/openscenario_preprocessor/package.xml index 9ef1d61bb98..ece50228050 100644 --- a/openscenario/openscenario_preprocessor/package.xml +++ b/openscenario/openscenario_preprocessor/package.xml @@ -24,7 +24,9 @@ ament_cmake_pep257 ament_cmake_xmllint ament_lint_auto - ament_cmake_gtest + ament_cmake_pytest + openscenario_utility + ament_index_python ament_cmake diff --git a/openscenario/openscenario_preprocessor/test/test_main.cpp b/openscenario/openscenario_preprocessor/test/test_main.cpp deleted file mode 100644 index 26afaf4e5bd..00000000000 --- a/openscenario/openscenario_preprocessor/test/test_main.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2015 TIER IV, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "gtest/gtest.h" - -#include - -auto dollar(const std::string & command) -> std::string -{ - std::array buffer; - - std::string result; - - std::unique_ptr pipe{::popen(command.c_str(), "r"), pclose}; - - if (not pipe) { - throw std::system_error(errno, std::system_category()); - } else { - while (std::fgets(buffer.data(), buffer.size(), pipe.get())) { - result += buffer.data(); - } - return result; - } -} - -TEST(getYawFromQuaternion, returnValue) -{ - std::stringstream preprocessor_command; - preprocessor_command << "ros2 run openscenario_preprocessor openscenario_preprocessor_command " - "-s /home/hans/Documents/sample.yaml " - "--parameters '{\"random_offset\": true}' " - "-f t4v2 -o /home/hans/Documents/cmd " - "--skip-full-derivation"; - dollar(preprocessor_command.str()); - - std::stringstream diff_command; - diff_command << " diff " - << "-bB " - << "/tmp/scenario_test_runner/0.0/0.0_0.xosc" - << " /tmp/openscenario_preprocessor/normalized.xosc" - << " | sed '/^[^>].*/d' | wc -l"; - std::string diff_result = dollar(diff_command.str()); - - EXPECT_EQ(1, boost::lexical_cast(diff_result)); -} - -auto main(int argc, char ** argv) -> int -{ - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/openscenario/openscenario_preprocessor/test/test_main.py b/openscenario/openscenario_preprocessor/test/test_main.py new file mode 100644 index 00000000000..7101c606037 --- /dev/null +++ b/openscenario/openscenario_preprocessor/test/test_main.py @@ -0,0 +1,50 @@ +from time import sleep + +import pytest +import os +from openscenario_utility.conversion import convert +from ament_index_python.packages import get_package_share_directory +from pathlib import Path +import difflib + + +@pytest.mark.skip +def test(scenario_name, desired_diff_num): + # share_dir = get_package_share_directory("openscenario_preprocessor") + preprocessor_share_dir = get_package_share_directory("openscenario_preprocessor") + test_runner_share_dir = preprocessor_share_dir + "/../../../scenario_test_runner/share/scenario_test_runner" + # sample_scenario_path = Path(share_dir + "/test/scenarios/" + scenario_name + ".yaml") + sample_scenario_path = Path(test_runner_share_dir + "/scenario/" + scenario_name + ".yaml") + output_dir = Path("/tmp/openscenario_preprocessor/test") + os.makedirs(output_dir, exist_ok=True) + convert(sample_scenario_path, output_dir) + input_xosc = "/tmp/openscenario_preprocessor/test/" + scenario_name + "_0.xosc" + + command_path = preprocessor_share_dir + "/../../lib/openscenario_preprocessor/openscenario_preprocessor_command" + preprocessor_command = command_path + " -s " + str(sample_scenario_path) + " -o /tmp/openscenario_preprocessor/test" \ + + " --parameters '{\"random_offset\": true}'" + " -f t4v2" + " --skip-full-derivation" + print(preprocessor_command) + os.system(preprocessor_command) + sleep(1) + + file1 = open(input_xosc) + file2 = open("/tmp/openscenario_preprocessor/normalized.xosc") + diff = difflib.Differ() + output_diff = diff.compare(file1.readlines(), file2.readlines()) + diff_num = 0 + for line in output_diff: + if line[0:1] in ['+', '-']: + diff_num += 1 + assert diff_num == desired_diff_num + + +def test_sample(): + test("sample", 2) + + +def test_autoware_simple(): + test("autoware-simple", 2) + + +def test_failure(): + test("failure", 4) From 6da23f46c1516a86b055837d6e5fb9cf43678bcb Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Tue, 21 Nov 2023 23:37:35 +0900 Subject: [PATCH 43/52] chore(cspell): update ignorePaths in custom_spell.json --- .github/workflows/custom_spell.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom_spell.json b/.github/workflows/custom_spell.json index edce0beb08f..e361eccb745 100644 --- a/.github/workflows/custom_spell.json +++ b/.github/workflows/custom_spell.json @@ -1,5 +1,5 @@ { - "ignorePaths": ["*.drawio", "*/**/CHANGELOG.rst", "external/lanelet2_matching/**", "*.xsd"], + "ignorePaths": ["*.drawio", "*/**/CHANGELOG.rst", "*.xsd"], "words": [ "ARGN", "AUTORCC", From cfd76ee1c70b85d7428a2e7057f0cef99ba69bb4 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Tue, 7 May 2024 16:23:24 +0900 Subject: [PATCH 44/52] fix: fix compile errors --- openscenario/openscenario_interpreter/CMakeLists.txt | 1 + openscenario/openscenario_preprocessor/src/t4v2.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openscenario/openscenario_interpreter/CMakeLists.txt b/openscenario/openscenario_interpreter/CMakeLists.txt index e78fa08e356..2db1e7b8d61 100644 --- a/openscenario/openscenario_interpreter/CMakeLists.txt +++ b/openscenario/openscenario_interpreter/CMakeLists.txt @@ -83,6 +83,7 @@ add_library(parameter_value_distribution_without_ros STATIC src/syntax/histogram.cpp src/syntax/histogram_bin.cpp src/syntax/normal_distribution.cpp + src/syntax/log_normal_distribution.cpp src/syntax/poisson_distribution.cpp src/syntax/probability_distribution_set.cpp src/syntax/probability_distribution_set_element.cpp diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index 4b5dd4f0607..ee8cbf1dce9 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -293,14 +293,14 @@ auto T4V2::deriveScenarioWithScenarioModifiers( struct replace_walker : pugi::xml_tree_walker { - openscenario_interpreter::ParameterListSharedPtr parameter_list; + openscenario_interpreter::ParameterSetSharedPtr parameter_set; virtual bool for_each(pugi::xml_node & node) { for (pugi::xml_attribute_iterator attribute_iter = node.attributes_begin(); attribute_iter != node.attributes_end(); ++attribute_iter) { - for (const auto & parameter : *parameter_list) { - attribute_iter->set_value(regex_replace( + for (const auto & parameter : *parameter_set) { + attribute_iter->set_value(std::regex_replace( attribute_iter->as_string(), std::regex(parameter.first), parameter.second.as()) .c_str()); @@ -312,13 +312,13 @@ auto T4V2::deriveScenarioWithScenarioModifiers( replace_walker walker; - for (const auto & parameter_list : scenario_modifier_distribution | boost::adaptors::indexed()) { + for (const auto & parameter_set : scenario_modifier_distribution | boost::adaptors::indexed()) { pugi::xml_document derived_script; derived_script.reset(base_scenario_doc); // deep copy // overwrite with scenario modifiers - walker.parameter_list = parameter_list.value(); + walker.parameter_set = parameter_set.value(); derived_script.traverse(walker); derived_scripts.push_back(std::forward(derived_script)); From acca4d91e991b3d064811eb03e62da9820f75857 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Thu, 16 May 2024 15:59:36 +0900 Subject: [PATCH 45/52] fix: fix build and link errors in openscenario_preprocessor --- openscenario/openscenario_preprocessor/CMakeLists.txt | 2 +- openscenario/openscenario_preprocessor/src/t4v2.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 49f3408767f..76c32dc6842 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -47,7 +47,7 @@ target_link_libraries(${PROJECT_NAME}_command pugixml) target_link_libraries(${PROJECT_NAME}_command yaml-cpp) target_link_libraries(${PROJECT_NAME}_command ${parameter_value_distribution_LIBRARIES}) -target_link_libraries(${PROJECT_NAME}_command ${openscenario_validator_LIBRARIES}) +target_link_libraries(${PROJECT_NAME}_command openscenario_validator::openscenario_validator) install(TARGETS ${PROJECT_NAME}_node ${PROJECT_NAME}_command RUNTIME DESTINATION lib/${PROJECT_NAME} diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index ee8cbf1dce9..aff15f8d8d8 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include From 0fc6b1efda740c412fd17f412f781ee5d7c6779b Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 17 May 2024 16:17:52 +0900 Subject: [PATCH 46/52] fix: add license notice in openscenario_preprocessor/test/test_main.py --- .../openscenario_preprocessor/test/test_main.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/openscenario/openscenario_preprocessor/test/test_main.py b/openscenario/openscenario_preprocessor/test/test_main.py index 7101c606037..e5a8c7b1712 100644 --- a/openscenario/openscenario_preprocessor/test/test_main.py +++ b/openscenario/openscenario_preprocessor/test/test_main.py @@ -1,3 +1,17 @@ +# Copyright 2015 TIER IV, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from time import sleep import pytest From f3e2f76c3917ab6d28dd35043bb934cd802fb258 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Fri, 17 May 2024 16:24:35 +0900 Subject: [PATCH 47/52] fix: delete unavailable test for openscenario_preprocessor --- openscenario/openscenario_preprocessor/test/test_main.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openscenario/openscenario_preprocessor/test/test_main.py b/openscenario/openscenario_preprocessor/test/test_main.py index e5a8c7b1712..7aecfd6142d 100644 --- a/openscenario/openscenario_preprocessor/test/test_main.py +++ b/openscenario/openscenario_preprocessor/test/test_main.py @@ -58,7 +58,3 @@ def test_sample(): def test_autoware_simple(): test("autoware-simple", 2) - - -def test_failure(): - test("failure", 4) From 78a48471f6b1861452264885996abaac83e8d2da Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 22 May 2024 19:20:41 +0900 Subject: [PATCH 48/52] refactor: delete ament_cpp_index dependencies from openscenario_validator as a library --- openscenario/openscenario_validator/CMakeLists.txt | 2 +- .../include/openscenario_validator/validator.hpp | 8 ++------ .../openscenario_validator/src/validator_command.cpp | 5 ++++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/openscenario/openscenario_validator/CMakeLists.txt b/openscenario/openscenario_validator/CMakeLists.txt index 75e8f4eab1d..12084f2ad06 100644 --- a/openscenario/openscenario_validator/CMakeLists.txt +++ b/openscenario/openscenario_validator/CMakeLists.txt @@ -15,7 +15,7 @@ target_include_directories(validate PUBLIC $) ament_export_include_directories(include) -ament_export_dependencies(XercesC ament_index_cpp) +ament_export_dependencies(XercesC) install(DIRECTORY include/ DESTINATION include) diff --git a/openscenario/openscenario_validator/include/openscenario_validator/validator.hpp b/openscenario/openscenario_validator/include/openscenario_validator/validator.hpp index c39c9fdb129..0a685933bde 100644 --- a/openscenario/openscenario_validator/include/openscenario_validator/validator.hpp +++ b/openscenario/openscenario_validator/include/openscenario_validator/validator.hpp @@ -15,7 +15,6 @@ #ifndef OPENSCENARIO_VALIDATOR__VALIDATOR_HPP_ #define OPENSCENARIO_VALIDATOR__VALIDATOR_HPP_ -#include #include #include #include @@ -74,17 +73,14 @@ class OpenSCENARIOValidator static inline XMLPlatformLifecycleHandler xml_platform_lifecycle_handler; public: - OpenSCENARIOValidator() : parser(std::make_unique()) + explicit OpenSCENARIOValidator(std::string schema_path) + : parser(std::make_unique()) { parser->setDoNamespaces(true); parser->setDoSchema(true); parser->setErrorHandler(&error_handler); parser->setValidationSchemaFullChecking(true); parser->setValidationScheme(xercesc::XercesDOMParser::Val_Always); - - std::string schema_path = - ament_index_cpp::get_package_share_directory("openscenario_validator") + - "/schema/OpenSCENARIO-1.3.xsd"; parser->setExternalNoNamespaceSchemaLocation(schema_path.c_str()); } diff --git a/openscenario/openscenario_validator/src/validator_command.cpp b/openscenario/openscenario_validator/src/validator_command.cpp index 147de8d6387..de2e35cc27e 100644 --- a/openscenario/openscenario_validator/src/validator_command.cpp +++ b/openscenario/openscenario_validator/src/validator_command.cpp @@ -12,13 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include int main(int argc, char * argv[]) { assert(argc == 2); std::string file_path = argv[1]; - openscenario_validator::OpenSCENARIOValidator validate; + std::string schema_path = ament_index_cpp::get_package_share_directory("openscenario_validator") + + "/schema/OpenSCENARIO-1.3.xsd"; + openscenario_validator::OpenSCENARIOValidator validate(schema_path); try { validate(file_path); return 0; From 9cd216b34571c4d0b93ede6d960a2664b16821f9 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 22 May 2024 20:12:06 +0900 Subject: [PATCH 49/52] refactor: setup in-binary schema in openscenario_preprocessor --- .../openscenario_preprocessor/CMakeLists.txt | 5 +++++ .../configure/schema.cpp | 20 +++++++++++++++++++ .../openscenario_preprocessor/schema.hpp | 18 +++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 openscenario/openscenario_preprocessor/configure/schema.cpp create mode 100644 openscenario/openscenario_preprocessor/include/openscenario_preprocessor/schema.hpp diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 882bbc4ee62..7b90542cc09 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -19,6 +19,7 @@ find_package(pugixml REQUIRED) ament_auto_find_build_dependencies() ament_auto_add_executable(${PROJECT_NAME}_node + src/schema.cpp src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_node.cpp) @@ -27,8 +28,12 @@ target_link_libraries(${PROJECT_NAME}_node yaml-cpp) find_library(parameter_value_distribution_LIBRARIES parameter_value_distribution_without_ros) +file(READ ${openscenario_validator_DIR}/../../openscenario_validator/schema/OpenSCENARIO-1.3.xsd ${PROJECT_NAME}_OPENSCENARIO_1_3_XSD) +configure_file(configure/schema.cpp src/schema.cpp) + add_executable(${PROJECT_NAME}_command src/t4v2.cpp + src/schema.cpp src/${PROJECT_NAME}.cpp src/${PROJECT_NAME}_command.cpp) diff --git a/openscenario/openscenario_preprocessor/configure/schema.cpp b/openscenario/openscenario_preprocessor/configure/schema.cpp new file mode 100644 index 00000000000..badb5ab3460 --- /dev/null +++ b/openscenario/openscenario_preprocessor/configure/schema.cpp @@ -0,0 +1,20 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace openscenario_preprocessor +{ + const char schema[] = R"###(${${PROJECT_NAME}_OPENSCENARIO_1_3_XSD})###"; +} // namespace openscenario_preprocessor diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/schema.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/schema.hpp new file mode 100644 index 00000000000..5ef5ea7c39a --- /dev/null +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/schema.hpp @@ -0,0 +1,18 @@ +// Copyright 2015 TIER IV, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +namespace openscenario_preprocessor +{ +extern const char schema[]; +} // namespace openscenario_preprocessor From 092b98870d8f7fcb3d836ac36c2e28f47fddb966 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 22 May 2024 20:15:07 +0900 Subject: [PATCH 50/52] refactor: use in-binary schema in openscenario_preprocessor --- openscenario/openscenario_preprocessor/CMakeLists.txt | 1 - .../include/openscenario_preprocessor/deriver.hpp | 2 +- .../openscenario_preprocessor.hpp | 8 +++++++- .../src/openscenario_preprocessor_command.cpp | 6 ------ openscenario/openscenario_preprocessor/src/t4v2.cpp | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/openscenario/openscenario_preprocessor/CMakeLists.txt b/openscenario/openscenario_preprocessor/CMakeLists.txt index 7b90542cc09..cc3fdece02a 100644 --- a/openscenario/openscenario_preprocessor/CMakeLists.txt +++ b/openscenario/openscenario_preprocessor/CMakeLists.txt @@ -53,7 +53,6 @@ target_link_libraries(${PROJECT_NAME}_command pugixml) target_link_libraries(${PROJECT_NAME}_command yaml-cpp) target_link_libraries(${PROJECT_NAME}_command ${parameter_value_distribution_LIBRARIES}) -target_link_libraries(${PROJECT_NAME}_command openscenario_validator::openscenario_validator) install(TARGETS ${PROJECT_NAME}_node ${PROJECT_NAME}_command RUNTIME DESTINATION lib/${PROJECT_NAME} diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp index f98da37ce59..8fd360309c5 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/deriver.hpp @@ -27,7 +27,7 @@ namespace openscenario_preprocessor class Deriver { public: - Deriver() {} + explicit Deriver(std::string schema_path) : validate(schema_path) {} auto operator()(boost::filesystem::path path, bool check_scenario_path = true) -> openscenario_interpreter::ParameterDistribution { diff --git a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp index 50435cb3d99..6a7ddd3e81a 100644 --- a/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp +++ b/openscenario/openscenario_preprocessor/include/openscenario_preprocessor/openscenario_preprocessor.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -50,7 +51,12 @@ class Preprocessor { public: explicit Preprocessor(const boost::filesystem::path & output_directory) - : output_directory(output_directory) + : output_directory(output_directory), derive([]() -> std::string { + auto file = std::ofstream("/tmp/openscenario_preprocessor/schema.xsd", std::ios::trunc); + file << openscenario_preprocessor::schema; + file.close(); + return "/tmp/openscenario_preprocessor/schema.xsd"; + }()) { if (not boost::filesystem::exists(output_directory)) { boost::filesystem::create_directories(output_directory); diff --git a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp index 0d6f582dc65..83b00b0b183 100644 --- a/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp +++ b/openscenario/openscenario_preprocessor/src/openscenario_preprocessor_command.cpp @@ -17,7 +17,6 @@ #include #include #include -#include const std::string_view template_scenario = R"###( @@ -101,11 +100,6 @@ try { boost::filesystem::path scenario_modifiers_path{}; - // setup xsd file - auto file = std::ofstream("/tmp/openscenario_preprocessor/schema.xsd", std::ios::trunc); - file << openscenario_validator::schema; - file.close(); - // preprocess t4v2 format and convert to xosc scenarios if (scenario_path.extension() == ".yaml" or scenario_path.extension() == ".yml") { openscenario_preprocessor::T4V2 t4v2; diff --git a/openscenario/openscenario_preprocessor/src/t4v2.cpp b/openscenario/openscenario_preprocessor/src/t4v2.cpp index aff15f8d8d8..f3414a1fcff 100644 --- a/openscenario/openscenario_preprocessor/src/t4v2.cpp +++ b/openscenario/openscenario_preprocessor/src/t4v2.cpp @@ -141,7 +141,7 @@ auto T4V2::deriveToXoscStringScenarios( "/tmp/openscenario_preprocessor/scenario_modifiers_distribution.xosc"; parameter_value_distribution.save_file(parameter_value_distribution_path.string().c_str()); - Deriver derive; + Deriver derive("/tmp/openscenario_preprocessor/schema.xsd"); auto distribution = derive(parameter_value_distribution_path, false); if (not distribution.empty()) { From bb95ee270f690a5f12d103725b73c7daa24321ce Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Mon, 10 Jun 2024 20:33:07 +0900 Subject: [PATCH 51/52] chore: apply linter --- openscenario/openscenario_preprocessor/configure/schema.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openscenario/openscenario_preprocessor/configure/schema.cpp b/openscenario/openscenario_preprocessor/configure/schema.cpp index badb5ab3460..10ea52a163b 100644 --- a/openscenario/openscenario_preprocessor/configure/schema.cpp +++ b/openscenario/openscenario_preprocessor/configure/schema.cpp @@ -16,5 +16,5 @@ namespace openscenario_preprocessor { - const char schema[] = R"###(${${PROJECT_NAME}_OPENSCENARIO_1_3_XSD})###"; +const char schema[] = R"###(${${PROJECT_NAME}_OPENSCENARIO_1_3_XSD})###"; } // namespace openscenario_preprocessor From 3b2693c404554913e9a14e6142517f928e15e282 Mon Sep 17 00:00:00 2001 From: Kotaro Yoshimoto Date: Wed, 13 Nov 2024 02:42:10 +0000 Subject: [PATCH 52/52] fix: add Boost::JSON to parameter_value_distribution_without_ros --- openscenario/openscenario_interpreter/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openscenario/openscenario_interpreter/CMakeLists.txt b/openscenario/openscenario_interpreter/CMakeLists.txt index 85bbb490e48..a1afbcc7105 100644 --- a/openscenario/openscenario_interpreter/CMakeLists.txt +++ b/openscenario/openscenario_interpreter/CMakeLists.txt @@ -154,9 +154,11 @@ add_definitions("-DBOOST_ALLOW_DEPRECATED_HEADERS") # cspell: ignore DPARAMETER target_compile_definitions(parameter_value_distribution_without_ros PUBLIC -DPARAMETER_VALUE_DISTRIBUTION_ONLY) target_include_directories(parameter_value_distribution_without_ros PUBLIC include) +target_include_directories(parameter_value_distribution_without_ros PUBLIC ${boost_json_SOURCE_DIR}/include) target_include_directories(parameter_value_distribution_without_ros PUBLIC ${scenario_simulator_exception_INCLUDE_DIRS}) target_link_libraries(parameter_value_distribution_without_ros Boost::filesystem + Boost::json pugixml) install(