From 951e592af3b93e386edd0056d495d945f85e9133 Mon Sep 17 00:00:00 2001 From: Michael Heimpold Date: Fri, 21 Jun 2024 13:43:26 +0200 Subject: [PATCH 1/2] GenericPowermeter: improve logging and fix Modbus range (#730) * GenericPowerMeter: fix allowed range for Modbus device address The Modbus RTU address range only permits addresses from 1 to 247 (at least what makes sense in our context here). Signed-off-by: Michael Heimpold * GenericPowermeter: log lost and restored communication This introduces a high-level/device specific logging in case communication with power meter is failing or communication is restored again. Signed-off-by: Michael Heimpold --------- Signed-off-by: Michael Heimpold --- modules/GenericPowermeter/main/powermeterImpl.cpp | 13 +++++++++++++ modules/GenericPowermeter/main/powermeterImpl.hpp | 3 +++ modules/GenericPowermeter/manifest.yaml | 4 ++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/modules/GenericPowermeter/main/powermeterImpl.cpp b/modules/GenericPowermeter/main/powermeterImpl.cpp index a821d27755..ca96d468d9 100644 --- a/modules/GenericPowermeter/main/powermeterImpl.cpp +++ b/modules/GenericPowermeter/main/powermeterImpl.cpp @@ -376,6 +376,12 @@ void powermeterImpl::process_response(const RegisterData& register_data, const types::serial_comm_hub_requests::Result exponent_message) { if (register_message.status_code == types::serial_comm_hub_requests::StatusCodeEnum::Success) { + // in case the meter was unavailable before and now the query succeeded, + // we can tell the user about this good news and reset our flag + if (meter_is_unavailable) { + EVLOG_info << "Communication with power meter restored."; + meter_is_unavailable = false; + } int16_t exponent{0}; if (exponent_message.value.has_value()) { @@ -497,6 +503,13 @@ void powermeterImpl::process_response(const RegisterData& register_data, } else { // error: message sending failed output_error_with_content(register_message); + + // let's warn the user about the meter's unavailability once only + // (since we keep trying communicating an 'error' is not justified) + if (!meter_is_unavailable) { + EVLOG_warning << "Lost communication with power meter."; + meter_is_unavailable = true; + } } } diff --git a/modules/GenericPowermeter/main/powermeterImpl.hpp b/modules/GenericPowermeter/main/powermeterImpl.hpp index c22a7cd1ff..ced77b5f2d 100644 --- a/modules/GenericPowermeter/main/powermeterImpl.hpp +++ b/modules/GenericPowermeter/main/powermeterImpl.hpp @@ -108,6 +108,9 @@ class powermeterImpl : public powermeterImplBase { std::thread output_thread; + /// @brief Remember whether we already logged the meter's unavailability. + bool meter_is_unavailable{false}; + void init_default_values(); void init_register_assignments(const json& loaded_registers); int assign_register_data(const json& registers, const PowermeterRegisters register_type, diff --git a/modules/GenericPowermeter/manifest.yaml b/modules/GenericPowermeter/manifest.yaml index 1e0a3c1077..b675b5bdae 100644 --- a/modules/GenericPowermeter/manifest.yaml +++ b/modules/GenericPowermeter/manifest.yaml @@ -11,8 +11,8 @@ provides: powermeter_device_id: description: The powermeter's address on the serial bus type: integer - minimum: 0 - maximum: 255 + minimum: 1 + maximum: 247 default: 1 modbus_base_address: description: The base address for register access From d4257933d8ea161034db919a6f9cf43374a6d648 Mon Sep 17 00:00:00 2001 From: Andreas Heinrich Date: Sat, 22 Jun 2024 08:25:42 +0200 Subject: [PATCH 2/2] Refactor CI Workflow (#701) * Refactor cmake related to python packaging * Bump required `everest-cmake` version to `0.4` * Add cmake variables to configure python wheels (install prefix) * Run custom python commands in venv located in build dir * Setup `ev-cli` automatically by installing it from source in to venv Refactor CI * Split into multiple jobs: `Lint`, `Build and Test` and `Run Integration Tests` * Upload dist directory as artifact * Upload wheel files as artifact * Upload result.xml and report.html as artifact * Render result summary Signed-off-by: Andreas Heinrich --- .ci/build-kit/compile.sh | 9 +- .ci/build-kit/install.sh | 8 ++ .ci/build-kit/prepare_integration_tests.sh | 10 +- .ci/build-kit/run_unit_tests.sh | 4 +- .ci/e2e/docker-compose.yaml | 5 +- .ci/e2e/scripts/run_integration_tests.sh | 11 ++- .github/workflows/build_and_test.yaml | 101 ++++++++++++++++++--- CMakeLists.txt | 8 ++ cmake/ev-cli.cmake | 25 ++++- cmake/ev-project-bootstrap.cmake | 5 +- cmake/everest-generate.cmake | 3 + dependencies.yaml | 12 ++- third-party/bazel/deps_versions.bzl | 6 +- 13 files changed, 170 insertions(+), 37 deletions(-) create mode 100755 .ci/build-kit/install.sh diff --git a/.ci/build-kit/compile.sh b/.ci/build-kit/compile.sh index cedc731153..dc21819268 100755 --- a/.ci/build-kit/compile.sh +++ b/.ci/build-kit/compile.sh @@ -1,12 +1,15 @@ #!/bin/sh +set -e + cmake \ - -B build \ + -B "$EXT_MOUNT/build" \ -S "$EXT_MOUNT/source" \ -G Ninja \ -DEVC_ENABLE_CCACHE=1 \ -DISO15118_2_GENERATE_AND_INSTALL_CERTIFICATES=OFF \ - -DCMAKE_INSTALL_PREFIX="$WORKSPACE_PATH/dist" \ + -DCMAKE_INSTALL_PREFIX="$EXT_MOUNT/dist" \ + -DWHEEL_INSTALL_PREFIX="$EXT_MOUNT/dist-wheels" \ -DBUILD_TESTING=ON -ninja -j$(nproc) -C build +ninja -j$(nproc) -C "$EXT_MOUNT/build" diff --git a/.ci/build-kit/install.sh b/.ci/build-kit/install.sh new file mode 100755 index 0000000000..5b78887564 --- /dev/null +++ b/.ci/build-kit/install.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e + +ninja -C "$EXT_MOUNT/build" install +ninja -C "$EXT_MOUNT/build" everestpy_install_wheel +ninja -C "$EXT_MOUNT/build" everest-testing_install_wheel +ninja -C "$EXT_MOUNT/build" iso15118_install_wheel diff --git a/.ci/build-kit/prepare_integration_tests.sh b/.ci/build-kit/prepare_integration_tests.sh index 7e19aaf95f..c083133c20 100755 --- a/.ci/build-kit/prepare_integration_tests.sh +++ b/.ci/build-kit/prepare_integration_tests.sh @@ -1,11 +1,9 @@ #!/bin/sh -# ninja -j$(nproc) -C build tests -ninja -j$(nproc) -C build install - -# install everest testing by cmake target to make sure using the version defined in dependencies.yaml -ninja -C build install_everest_testing +set -e rsync -a "$EXT_MOUNT/source/tests" ./ -rm -rf build +pip install $EXT_MOUNT/wheels/everestpy-*.whl +pip install $EXT_MOUNT/wheels/everest_testing-*.whl +pip install pytest-html diff --git a/.ci/build-kit/run_unit_tests.sh b/.ci/build-kit/run_unit_tests.sh index 5283d4420b..d73a4e53c5 100755 --- a/.ci/build-kit/run_unit_tests.sh +++ b/.ci/build-kit/run_unit_tests.sh @@ -1,3 +1,5 @@ #!/bin/sh -ninja -j$(nproc) -C build test +set -e + +ninja -C "$EXT_MOUNT/build" test diff --git a/.ci/e2e/docker-compose.yaml b/.ci/e2e/docker-compose.yaml index b6a5391e83..1d8ae11de0 100644 --- a/.ci/e2e/docker-compose.yaml +++ b/.ci/e2e/docker-compose.yaml @@ -18,7 +18,10 @@ services: - MQTT_SERVER_ADDRESS=mqtt-server volumes: - type: bind - source: ./scripts + source: ${GITHUB_WORKSPACE} + target: /ext + - type: bind + source: ${GITHUB_WORKSPACE}/source/.ci/e2e/scripts target: /ext/scripts sysctls: - net.ipv6.conf.all.disable_ipv6=0 diff --git a/.ci/e2e/scripts/run_integration_tests.sh b/.ci/e2e/scripts/run_integration_tests.sh index 9ddf2cf6f2..0ffbe34100 100755 --- a/.ci/e2e/scripts/run_integration_tests.sh +++ b/.ci/e2e/scripts/run_integration_tests.sh @@ -1,4 +1,13 @@ #!/bin/sh +set -e + cd tests -pytest --everest-prefix ../dist core_tests/*.py framework_tests/*.py +pytest \ + -rA \ + --junitxml="$EXT_MOUNT/result.xml" \ + --html="$EXT_MOUNT/report.html" \ + --self-contained-html \ + core_tests/*.py \ + framework_tests/*.py \ + --everest-prefix "$EXT_MOUNT/dist" diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index a9526cac0c..7f7caee51c 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -15,11 +15,25 @@ on: - cron: '37 13,1 * * *' env: - BUILD_KIT_IMAGE: ghcr.io/everest/build-kit-alpine:v1.1.4 + BUILD_KIT_IMAGE: ghcr.io/everest/build-kit-alpine:v1.2.0 jobs: + lint: + name: Lint + runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} + steps: + - name: Checkout everest-core + uses: actions/checkout@v4.1.6 + with: + path: source + - name: Run clang-format + uses: everest/everest-ci/github-actions/run-clang-format@v1.1.0 + with: + source-dir: source + extensions: hpp,cpp + exclude: cache build: - name: Build, Lint and Test + name: Build and Unit Tests runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} steps: - name: Format branch name for cache key @@ -35,15 +49,9 @@ jobs: compile-${{ env.branch_name_for_cache }}- compile- - name: Checkout everest-core - uses: actions/checkout@v3 + uses: actions/checkout@v4.1.6 with: path: source - - name: Run clang-format - uses: everest/everest-ci/github-actions/run-clang-format@v1.1.0 - with: - source-dir: source - extensions: hpp,cpp - exclude: cache - name: Setup run scripts run: | mkdir scripts @@ -67,14 +75,81 @@ jobs: --volume "$(pwd):/ext" \ --name unit-tests-container \ build-image run-script run_unit_tests + - name: Create dist + run: | + docker run \ + --volume "$(pwd):/ext" \ + --name install-container \ + build-image run-script install + - name: Tar dist dir and keep permissions + run: | + tar -czf dist.tar.gz dist + - name: Upload dist artifact + uses: actions/upload-artifact@v4.3.3 + with: + path: dist.tar.gz + name: dist + - name: Upload wheels artifact + uses: actions/upload-artifact@v4.3.3 + with: + path: dist-wheels + name: wheels + integration-tests: + name: Integration Tests + needs: build + runs-on: ${{ inputs.runner || 'ubuntu-22.04' }} + steps: + - name: Download dist dir + uses: actions/download-artifact@v4.1.2 + with: + name: dist + - name: Extract dist.tar.gz + run: | + tar -xzf ${{ github.workspace }}/dist.tar.gz -C ${{ github.workspace }} + - name: Download wheels + uses: actions/download-artifact@v4.1.2 + with: + name: wheels + path: wheels + - name: Checkout everest-core + uses: actions/checkout@v4.1.6 + with: + path: source + - name: Setup run scripts + run: | + mkdir scripts + rsync -a source/.ci/build-kit/ scripts + - name: Pull build-kit image + run: | + docker pull --quiet ${{ env.BUILD_KIT_IMAGE }} + docker image tag ${{ env.BUILD_KIT_IMAGE }} build-kit - name: Create integration-image run: | docker run \ --volume "$(pwd):/ext" \ - --name test-container \ - build-image run-script prepare_integration_tests - docker commit test-container integration-image + --name prepare-container \ + build-kit run-script prepare_integration_tests + docker commit prepare-container integration-image - name: Run integration tests run: | pushd source/.ci/e2e - docker-compose run e2e-test-server run-script run_integration_tests + docker-compose run \ + e2e-test-server \ + run-script run_integration_tests + - name: Upload result & report as artifact + if: always() + uses: actions/upload-artifact@v4.3.3 + with: + path: | + ${{ github.workspace }}/result.xml + ${{ github.workspace }}/report.html + name: pytest-results + - name: Render result + if: always() + uses: pmeier/pytest-results-action@v0.6.0 + with: + path: ${{ github.workspace }}/result.xml + summary: True + display-options: fEX + fail-on-empty: True + title: Test results diff --git a/CMakeLists.txt b/CMakeLists.txt index 70943b9a54..caab156a97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,14 @@ endif() # This is a flag for building development tests, but not necessarily to run them, for expample in case # tests requires hardware. option(BUILD_DEV_TESTS "Build dev tests" OFF) +ev_setup_cmake_variables_python_wheel() +option(${PROJECT_NAME}_INSTALL_EV_CLI_IN_PYTHON_VENV "Install ev-cli in python venv instead of using system" ON) +set(${PROJECT_NAME}_PYTHON_VENV_PATH "${CMAKE_BINARY_DIR}/venv" CACHE PATH "Path to python venv") + +ev_setup_python_executable( + USE_PYTHON_VENV ${${PROJECT_NAME}_USE_PYTHON_VENV} + PYTHON_VENV_PATH ${${PROJECT_NAME}_PYTHON_VENV_PATH} +) # Already include CTest here to allow it to find tests defined in subdirectories like lib and modules if(EVEREST_CORE_BUILD_TESTING) diff --git a/cmake/ev-cli.cmake b/cmake/ev-cli.cmake index d4a196fb89..1305a23fcb 100644 --- a/cmake/ev-cli.cmake +++ b/cmake/ev-cli.cmake @@ -1,7 +1,24 @@ - -# check ev-cli version - -find_program(EV_CLI ev-cli REQUIRED) +macro(setup_ev_cli) + add_custom_target(ev-cli) + if(${EV_CLI}) + message(FATAL_ERROR "EV_CLI is already defined.") + return() + endif() + if(NOT ${${PROJECT_NAME}_USE_PYTHON_VENV}) + find_program(EV_CLI ev-cli REQUIRED) + return() + endif() + ev_is_python_venv_active( + RESULT_VAR IS_PYTHON_VENV_ACTIVE + ) + if(NOT ${IS_PYTHON_VENV_ACTIVE}) + message(FATAL_ERROR "Python venv is not active. Please activate the python venv before running this command.") + endif() + set(EV_CLI "${${PROJECT_NAME}_PYTHON_VENV_PATH}/bin/ev-cli") + add_dependencies(ev-cli + ev-dev-tools_pip_install_dist + ) +endmacro() function(require_ev_cli_version EV_CLI_VERSION_REQUIRED) execute_process( diff --git a/cmake/ev-project-bootstrap.cmake b/cmake/ev-project-bootstrap.cmake index 41b3912ef7..324c9d3c1e 100644 --- a/cmake/ev-project-bootstrap.cmake +++ b/cmake/ev-project-bootstrap.cmake @@ -5,7 +5,10 @@ include(${CMAKE_CURRENT_LIST_DIR}/config-run-script.cmake) include(${CMAKE_CURRENT_LIST_DIR}/config-run-nodered-script.cmake) # dependencies -require_ev_cli_version("0.2.0") +setup_ev_cli() +if(NOT ${${PROJECT_NAME}_INSTALL_EV_CLI_IN_PYTHON_VENV}) + require_ev_cli_version("0.2.0") +endif() # source generate scripts / setup include(${CMAKE_CURRENT_LIST_DIR}/everest-generate.cmake) diff --git a/cmake/everest-generate.cmake b/cmake/everest-generate.cmake index ed31d0589a..75de8bda66 100644 --- a/cmake/everest-generate.cmake +++ b/cmake/everest-generate.cmake @@ -284,6 +284,7 @@ function (_ev_add_interfaces) "${CHECK_DONE_FILE}" DEPENDS ${ARGV} + ev-cli COMMENT "Generating/updating interface files ..." VERBATIM @@ -322,6 +323,7 @@ function (_ev_add_types) "${CHECK_DONE_FILE}" DEPENDS ${ARGV} + ev-cli COMMENT "Generating/updating type files ..." VERBATIM @@ -465,6 +467,7 @@ function (ev_add_cpp_module MODULE_NAME) ${RELATIVE_MODULE_DIR} DEPENDS ${MODULE_PATH}/manifest.yaml + ev-cli WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMENT diff --git a/dependencies.yaml b/dependencies.yaml index 332319afd6..c69f8cb7c5 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -4,8 +4,12 @@ --- everest-framework: git: https://github.com/EVerest/everest-framework.git - git_tag: v0.15.0 - options: ["BUILD_TESTING OFF"] + git_tag: v0.15.1 + options: [ + "BUILD_TESTING OFF", + "everest-framework_USE_PYTHON_VENV ${PROJECT_NAME}_USE_PYTHON_VENV", + "everest-framework_USE_PYTHON_VENV_PATH ${PROJECT_NAME}_USE_PYTHON_VENV_PATH", + ] sigslot: git: https://github.com/palacaze/sigslot git_tag: v1.2.0 @@ -61,7 +65,7 @@ libocpp: # Josev Josev: git: https://github.com/EVerest/ext-switchev-iso15118.git - git_tag: 2024.2.1 + git_tag: 2024.6.0 cmake_condition: "EVEREST_ENABLE_PY_SUPPORT AND EVEREST_DEPENDENCY_ENABLED_JOSEV" # OpenV2G ext-openv2g: @@ -80,7 +84,7 @@ ext-mbedtls: # everest-testing everest-utils: git: https://github.com/EVerest/everest-utils.git - git_tag: v0.2.3 + git_tag: v0.3.0 cmake_condition: "EVEREST_CORE_BUILD_TESTING" # unit testing diff --git a/third-party/bazel/deps_versions.bzl b/third-party/bazel/deps_versions.bzl index b63303d110..88a9772b31 100644 --- a/third-party/bazel/deps_versions.bzl +++ b/third-party/bazel/deps_versions.bzl @@ -7,7 +7,7 @@ EVEREST_DEPS = struct( # Josev Josev_repo = "https://github.com/EVerest/ext-switchev-iso15118.git", Josev_commit = None, - Josev_tag = "2024.2.1", + Josev_tag = "2024.6.0", # catch2 catch2_repo = "https://github.com/catchorg/Catch2.git", @@ -17,12 +17,12 @@ EVEREST_DEPS = struct( # everest-framework everest_framework_repo = "https://github.com/EVerest/everest-framework.git", everest_framework_commit = None, - everest_framework_tag = "v0.15.0", + everest_framework_tag = "v0.15.1", # everest-utils everest_utils_repo = "https://github.com/EVerest/everest-utils.git", everest_utils_commit = None, - everest_utils_tag = "v0.2.3", + everest_utils_tag = "v0.3.0", # ext-mbedtls ext_mbedtls_repo = "https://github.com/EVerest/ext-mbedtls.git",