From 04ad161d7a42f70706f9f388d0e731dd39af42d8 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 18 Sep 2023 03:17:06 -0700 Subject: [PATCH 01/12] homebrew_formula_pullrequest.bash: sanitize ~ (#1007) Sanitize ~ characters from the brew version string. Signed-off-by: Steve Peters --- .../lib/homebrew_formula_pullrequest.bash | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash index e353f1f4a..0256f8eed 100644 --- a/jenkins-scripts/lib/homebrew_formula_pullrequest.bash +++ b/jenkins-scripts/lib/homebrew_formula_pullrequest.bash @@ -26,6 +26,10 @@ if [ -z "${SOURCE_TARBALL_SHA}" ]; then fi echo '# END SECTION' +# brew does not support ~ in VERSION string, so convert it to - +VERSION_SANITIZED=$(echo "${VERSION}" | tr '~' '-') +echo Sanitizing version string ${VERSION} to ${VERSION_SANITIZED} + PULL_REQUEST_HEAD_REPO=git@github.com:osrfbuild/homebrew-simulation.git . ${SCRIPT_LIBDIR}/_homebrew_github_setup.bash @@ -48,7 +52,7 @@ echo VERSION_LINE=$(awk \ "/^ version ['\"]/ {print FNR}" ${FORMULA_PATH} | head -1) # check if version can be correctly auto-detected from url -if ${BREW} ruby -e "exit Version.parse(\"${SOURCE_TARBALL_URI}\").to_s == \"${VERSION}\"" +if ${BREW} ruby -e "exit Version.parse(\"${SOURCE_TARBALL_URI}\").to_s == \"${VERSION_SANITIZED}\"" then echo Version can be correctly auto-detected from URL if [ -n "${VERSION_LINE}" ]; then @@ -59,12 +63,12 @@ else if [ -z "${VERSION_LINE}" ]; then # Need to insert explicit version tag after url echo Adding explicit version tag after URL - sed -i -e "${URI_LINE}a\ version \"${VERSION}\"" ${FORMULA_PATH} + sed -i -e "${URI_LINE}a\ version \"${VERSION_SANITIZED}\"" ${FORMULA_PATH} else echo Changing version to - echo ${VERSION} + echo ${VERSION_SANITIZED} echo on line number ${VERSION_LINE} - sed -i -e "${VERSION_LINE}c\ version \"${VERSION}\"" ${FORMULA_PATH} + sed -i -e "${VERSION_LINE}c\ version \"${VERSION_SANITIZED}\"" ${FORMULA_PATH} fi fi @@ -104,6 +108,6 @@ if [ "$FORMULA_REVISION" -gt 0 ]; then fi # create branch with name and sanitized version string -PULL_REQUEST_BRANCH="${PACKAGE_ALIAS}_`echo ${VERSION} | tr ' ~:^?*[' '_'`" +PULL_REQUEST_BRANCH="${PACKAGE_ALIAS}_`echo ${VERSION_SANITIZED} | tr ' ~:^?*[' '_'`" . ${SCRIPT_LIBDIR}/_homebrew_github_commit.bash From 87481352f1df1c65c3946f19b5edcb7b42c255f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Crist=C3=B3bal=20Arroyo?= Date: Tue, 19 Sep 2023 08:26:53 -0500 Subject: [PATCH 02/12] Update _outdated_job_runner jobs (#981) * Update jobs Signed-off-by: Crola1702 * Update pseudo-documentation Signed-off-by: Crola1702 * Update tracked jobs Signed-off-by: Crola1702 * Fix sdf13 pin Signed-off-by: Crola1702 --------- Signed-off-by: Crola1702 --- .../tools/outdated-job-runner.groovy | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/jenkins-scripts/tools/outdated-job-runner.groovy b/jenkins-scripts/tools/outdated-job-runner.groovy index bf030d162..bf9fc5e16 100644 --- a/jenkins-scripts/tools/outdated-job-runner.groovy +++ b/jenkins-scripts/tools/outdated-job-runner.groovy @@ -64,61 +64,75 @@ def runJobsInAvailableNodes(LinkedHashMap outdatedJobs) { } // TODO (Cristobal): This is a hardcoded list of jobs that we're tracking. We should get this list dynamically -// Currently is the result of https://github.com/osrf/buildfarmer/blob/main/common.py#L51 minus debbuilders +// Currently is the result of https://github.com/osrf/buildfarmer/blob/main/scripts/gazebo_non_nightly_jobs.py def trackedJobsList = [ + 'gz_cmake-ci-gz-cmake3-jammy-amd64', + 'gz_common-ci-gz-common5-jammy-amd64', + 'gz_fuel_tools-ci-gz-fuel-tools9-jammy-amd64', + 'gz_gui-ci-gz-gui8-jammy-amd64', + 'gz_launch-ci-gz-launch7-jammy-amd64', + 'gz_math-ci-gz-math7-jammy-amd64', + 'gz_msgs-ci-gz-msgs10-jammy-amd64', + 'gz_physics-ci-gz-physics7-jammy-amd64', + 'gz_plugin-ci-gz-plugin2-jammy-amd64', + 'gz_rendering-ci-gz-rendering8-jammy-amd64', + 'gz_sensors-ci-gz-sensors8-jammy-amd64', + 'gz_sim-ci-gz-sim8-jammy-amd64', + 'gz_tools-ci-gz-tools2-jammy-amd64', + 'gz_transport-ci-gz-transport13-jammy-amd64', + 'gz_utils-ci-gz-utils2-jammy-amd64', 'ign_cmake-gz-3-win', 'ign_cmake-ign-2-win', 'ign_common-gz-5-win', 'ign_common-ign-3-win', 'ign_common-ign-4-win', - 'ign_fuel-tools-ci-win', 'ign_fuel-tools-gz-8-win', + 'ign_fuel-tools-gz-9-win', 'ign_fuel-tools-ign-4-win', 'ign_fuel-tools-ign-7-win', - 'ign_gazebo-ci-win', 'ign_gazebo-gz-7-win', + 'ign_gazebo-gz-8-win', 'ign_gazebo-ign-3-win', 'ign_gazebo-ign-6-win', - 'ign_gui-ci-win', 'ign_gui-gz-7-win', + 'ign_gui-gz-8-win', 'ign_gui-ign-3-win', 'ign_gui-ign-6-win', - 'ign_launch-ci-win', 'ign_launch-gz-6-win', + 'ign_launch-gz-7-win', 'ign_launch-ign-2-win', 'ign_launch-ign-5-win', 'ign_math-gz-7-win', 'ign_math-ign-6-win', - 'ign_msgs-ci-win', + 'ign_msgs-gz-10-win', 'ign_msgs-gz-9-win', 'ign_msgs-ign-5-win', 'ign_msgs-ign-8-win', 'ign_physics-gz-6-win', + 'ign_physics-gz-7-win', 'ign_physics-ign-2-win', 'ign_physics-ign-5-win', 'ign_plugin-gz-2-win', 'ign_plugin-ign-1-win', - 'ign_rendering-ci-win', 'ign_rendering-gz-7-win', + 'ign_rendering-gz-8-win', 'ign_rendering-ign-3-win', 'ign_rendering-ign-6-win', - 'ign_sensors-ci-win', 'ign_sensors-gz-7-win', + 'ign_sensors-gz-8-win', 'ign_sensors-ign-3-win', 'ign_sensors-ign-6-win', 'ign_tools-gz-2-win', 'ign_tools-ign-1-win', - 'ign_transport-ci-win', 'ign_transport-gz-12-win', + 'ign_transport-gz-13-win', 'ign_transport-ign-11-win', 'ign_transport-ign-8-win', 'ign_utils-gz-2-win', 'ign_utils-ign-1-win', - 'ignition_cmake-ci-gz-cmake3-focal-amd64', 'ignition_cmake-ci-gz-cmake3-homebrew-amd64', 'ignition_cmake-ci-ign-cmake2-focal-amd64', 'ignition_cmake-ci-ign-cmake2-homebrew-amd64', - 'ignition_common-ci-gz-common5-focal-amd64', 'ignition_common-ci-gz-common5-homebrew-amd64', 'ignition_common-ci-ign-common3-focal-amd64', 'ignition_common-ci-ign-common3-homebrew-amd64', @@ -126,98 +140,90 @@ def trackedJobsList = [ 'ignition_common-ci-ign-common4-homebrew-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-focal-amd64', 'ignition_fuel-tools-ci-gz-fuel-tools8-homebrew-amd64', + 'ignition_fuel-tools-ci-gz-fuel-tools9-homebrew-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools4-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools4-homebrew-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-focal-amd64', 'ignition_fuel-tools-ci-ign-fuel-tools7-homebrew-amd64', - 'ignition_fuel-tools-ci-main-focal-amd64', - 'ignition_fuel-tools-ci-main-homebrew-amd64', 'ignition_gazebo-ci-gz-sim7-focal-amd64', 'ignition_gazebo-ci-gz-sim7-homebrew-amd64', + 'ignition_gazebo-ci-gz-sim8-homebrew-amd64', 'ignition_gazebo-ci-ign-gazebo3-focal-amd64', 'ignition_gazebo-ci-ign-gazebo3-homebrew-amd64', 'ignition_gazebo-ci-ign-gazebo6-focal-amd64', 'ignition_gazebo-ci-ign-gazebo6-homebrew-amd64', - 'ignition_gazebo-ci-main-focal-amd64', - 'ignition_gazebo-ci-main-homebrew-amd64', 'ignition_gui-ci-gz-gui7-focal-amd64', 'ignition_gui-ci-gz-gui7-homebrew-amd64', + 'ignition_gui-ci-gz-gui8-homebrew-amd64', 'ignition_gui-ci-ign-gui3-focal-amd64', 'ignition_gui-ci-ign-gui3-homebrew-amd64', 'ignition_gui-ci-ign-gui6-focal-amd64', 'ignition_gui-ci-ign-gui6-homebrew-amd64', - 'ignition_gui-ci-main-focal-amd64', - 'ignition_gui-ci-main-homebrew-amd64', 'ignition_launch-ci-gz-launch6-focal-amd64', 'ignition_launch-ci-gz-launch6-homebrew-amd64', + 'ignition_launch-ci-gz-launch7-homebrew-amd64', 'ignition_launch-ci-ign-launch2-focal-amd64', 'ignition_launch-ci-ign-launch2-homebrew-amd64', 'ignition_launch-ci-ign-launch5-focal-amd64', 'ignition_launch-ci-ign-launch5-homebrew-amd64', - 'ignition_launch-ci-main-focal-amd64', - 'ignition_launch-ci-main-homebrew-amd64', - 'ignition_math-ci-gz-math7-focal-amd64', 'ignition_math-ci-gz-math7-homebrew-amd64', 'ignition_math-ci-ign-math6-focal-amd64', 'ignition_math-ci-ign-math6-homebrew-amd64', + 'ignition_msgs-ci-gz-msgs10-homebrew-amd64', 'ignition_msgs-ci-gz-msgs9-focal-amd64', 'ignition_msgs-ci-gz-msgs9-homebrew-amd64', 'ignition_msgs-ci-ign-msgs5-focal-amd64', 'ignition_msgs-ci-ign-msgs5-homebrew-amd64', 'ignition_msgs-ci-ign-msgs8-focal-amd64', 'ignition_msgs-ci-ign-msgs8-homebrew-amd64', - 'ignition_msgs-ci-main-focal-amd64', - 'ignition_msgs-ci-main-homebrew-amd64', 'ignition_physics-ci-gz-physics6-focal-amd64', 'ignition_physics-ci-gz-physics6-homebrew-amd64', + 'ignition_physics-ci-gz-physics7-homebrew-amd64', 'ignition_physics-ci-ign-physics2-focal-amd64', 'ignition_physics-ci-ign-physics2-homebrew-amd64', 'ignition_physics-ci-ign-physics5-focal-amd64', 'ignition_physics-ci-ign-physics5-homebrew-amd64', - 'ignition_plugin-ci-gz-plugin2-focal-amd64', 'ignition_plugin-ci-gz-plugin2-homebrew-amd64', 'ignition_plugin-ci-ign-plugin1-focal-amd64', 'ignition_plugin-ci-ign-plugin1-homebrew-amd64', 'ignition_rendering-ci-gz-rendering7-focal-amd64', 'ignition_rendering-ci-gz-rendering7-homebrew-amd64', + 'ignition_rendering-ci-gz-rendering8-homebrew-amd64', 'ignition_rendering-ci-ign-rendering3-focal-amd64', 'ignition_rendering-ci-ign-rendering3-homebrew-amd64', 'ignition_rendering-ci-ign-rendering6-focal-amd64', 'ignition_rendering-ci-ign-rendering6-homebrew-amd64', - 'ignition_rendering-ci-main-focal-amd64', - 'ignition_rendering-ci-main-homebrew-amd64', 'ignition_sensors-ci-gz-sensors7-focal-amd64', 'ignition_sensors-ci-gz-sensors7-homebrew-amd64', + 'ignition_sensors-ci-gz-sensors8-homebrew-amd64', 'ignition_sensors-ci-ign-sensors3-focal-amd64', 'ignition_sensors-ci-ign-sensors3-homebrew-amd64', 'ignition_sensors-ci-ign-sensors6-focal-amd64', 'ignition_sensors-ci-ign-sensors6-homebrew-amd64', - 'ignition_sensors-ci-main-focal-amd64', - 'ignition_sensors-ci-main-homebrew-amd64', - 'ignition_tools-ci-gz-tools2-focal-amd64', 'ignition_tools-ci-gz-tools2-homebrew-amd64', 'ignition_tools-ci-ign-tools1-focal-amd64', 'ignition_tools-ci-ign-tools1-homebrew-amd64', 'ignition_transport-ci-gz-transport12-focal-amd64', 'ignition_transport-ci-gz-transport12-homebrew-amd64', + 'ignition_transport-ci-gz-transport13-homebrew-amd64', 'ignition_transport-ci-ign-transport11-focal-amd64', 'ignition_transport-ci-ign-transport11-homebrew-amd64', 'ignition_transport-ci-ign-transport8-focal-amd64', 'ignition_transport-ci-ign-transport8-homebrew-amd64', - 'ignition_transport-ci-main-focal-amd64', - 'ignition_transport-ci-main-homebrew-amd64', - 'ignition_utils-ci-gz-utils2-focal-amd64', 'ignition_utils-ci-gz-utils2-homebrew-amd64', 'ignition_utils-ci-ign-utils1-focal-amd64', 'ignition_utils-ci-ign-utils1-homebrew-amd64', + 'sdformat-ci-sdf14-jammy-amd64', 'sdformat-ci-sdformat12-focal-amd64', 'sdformat-ci-sdformat12-homebrew-amd64', 'sdformat-ci-sdformat13-focal-amd64', 'sdformat-ci-sdformat13-homebrew-amd64', + 'sdformat-ci-sdformat14-homebrew-amd64', 'sdformat-ci-sdformat9-focal-amd64', 'sdformat-ci-sdformat9-homebrew-amd64', 'sdformat-sdf-12-win', 'sdformat-sdf-13-win', + 'sdformat-sdf-14-win', 'sdformat-sdf-9-win', ]; def trackedJobs = new LinkedHashSet(trackedJobsList); From 8d891d89d3dcabd24d89e6d4e218b3e53a1455d5 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 20 Sep 2023 16:02:58 +0200 Subject: [PATCH 03/12] Fixes for reprepro-importer (#1015) * Do not use sudo * Do not hardcode GNUPGHOME Signed-off-by: Jose Luis Rivero --- jenkins-scripts/docker/reprepro_updater.bash | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/docker/reprepro_updater.bash b/jenkins-scripts/docker/reprepro_updater.bash index 580070b9a..3246350fd 100644 --- a/jenkins-scripts/docker/reprepro_updater.bash +++ b/jenkins-scripts/docker/reprepro_updater.bash @@ -40,13 +40,12 @@ git clone "${REPREPRO_GIT_REPO}" -b "${REPREPRO_GIT_BRANCH}" "${REPREPRO_REPO_PA echo '# END SECTION' export PYTHONPATH="${REPREPRO_REPO_PATH}/src" -export GNUPGHOME=/var/lib/jenkins/.gnupg echo '# BEGIN SECTION: run reprepro' cd "${REPREPRO_REPO_PATH}/scripts" -sudo -E bash -c "PYTHONPATH=${REPREPRO_REPO_PATH}/src python3 import_upstream.py ${REPREPRO_PARAMS} \ - ${UPLOAD_TO_REPO:-:_} \ - ${REPREPRO_REPO_PATH}/config/packages.osrfoundation.org/${REPREPRO_IMPORT_YAML_FILE}" +PYTHONPATH=${REPREPRO_REPO_PATH}/src python3 import_upstream.py ${REPREPRO_PARAMS} \ + "${UPLOAD_TO_REPO:-:_}" \ + "${REPREPRO_REPO_PATH}/config/packages.osrfoundation.org/${REPREPRO_IMPORT_YAML_FILE}" echo '# END SECTION' echo '#BEGIN: exit the venv' From 59d8bbc85af731ce6120e277e288d200d139c6ed Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 20 Sep 2023 19:45:46 +0200 Subject: [PATCH 04/12] Disable S3 uploads for testing (#1016) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/lib/repository_uploader.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins-scripts/lib/repository_uploader.bash b/jenkins-scripts/lib/repository_uploader.bash index 79e01c224..f45ee35b8 100644 --- a/jenkins-scripts/lib/repository_uploader.bash +++ b/jenkins-scripts/lib/repository_uploader.bash @@ -112,8 +112,8 @@ case ${UPLOAD_TO_REPO} in exit 1 fi ;; - "nightly") - # No uploads for nightly packages test runs + "nightly" | "testing") + # No uploads for nightly or testing packages test runs ENABLE_S3_UPLOAD=false ;; "none") From 76e716f48d11ffc98a7298ad082c2e5578ba5f4d Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 22 Sep 2023 17:07:51 +0200 Subject: [PATCH 05/12] Improve OSRFSourceCreation (#1008) * Make static parameters to look like that Use one option parameter to block jobs from changing names and repos and use class members to support static members. * Add an step to export TARBALL_NAME * Support for adding downstream: releasepy + repo_uploader * Added Global support to handle S3 paths * Added _test_repository_uploader job --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/_configs_/Globals.groovy | 24 ++++ .../dsl/_configs_/OSRFSourceCreation.groovy | 106 +++++++++++++++--- jenkins-scripts/dsl/test.dsl | 54 ++++++++- 3 files changed, 166 insertions(+), 18 deletions(-) diff --git a/jenkins-scripts/dsl/_configs_/Globals.groovy b/jenkins-scripts/dsl/_configs_/Globals.groovy index 1386da8b6..2adc32511 100644 --- a/jenkins-scripts/dsl/_configs_/Globals.groovy +++ b/jenkins-scripts/dsl/_configs_/Globals.groovy @@ -131,4 +131,28 @@ class Globals static String nontest_label(String original_label) { return "(${original_label}) && !test-instance" } + + static String get_canonical_package_name(String package_name) { + return package_name.replaceAll('\\d*$', '') + } + + static String _s3_releases_dir(String package_name) { + return get_canonical_package_name(package_name) + '/releases' + } + + static String _s3_build_tarball_name(String package_name, String version) { + // canonical_name + version + return package_name.replaceAll('\\d*$', '') + '-' + version + } + + static String s3_upload_tarball_path(String package_name) { + return 's3://osrf-distributions/' + _s3_releases_dir(package_name) + } + + // Not yet in use. Requires changing release.py + static String s3_download_uri(String package_name, String version) { + return 'https://osrf-distributions.s3.amazonaws.com/' + \ + _s3_releases_dir(package_name) + \ + _s3_build_tarball_name(package_name, version) + } } diff --git a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy index cb01aa186..a7eed1933 100644 --- a/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy +++ b/jenkins-scripts/dsl/_configs_/OSRFSourceCreation.groovy @@ -5,43 +5,48 @@ import _configs_.Globals class OSRFSourceCreation { + static String properties_file = "package_name.prop" + static String package_name = "" + static void addParameters(Job job, Map default_params = [:]) { + package_name = default_params.find{ it.key == "PACKAGE"}?.value + job.with { parameters { - stringParam("PACKAGE_NAME", - default_params.find{ it.key == "PACKAGE_NAME"}?.value, - "Software name (i.e gz-cmake3)") - stringParam("SOURCE_REPO_URI", - default_params.find{ it.key == "SOURCE_REPO_URI"}?.value, - "GitHub URI to release the sources from (i.e: https://github.com/gazebosim/gz-cmake.git)") + choiceParam('PACKAGE', + [default_params.find{ it.key == "PACKAGE"}?.value], + "Package name (can not be modified)") + choiceParam('SOURCE_REPO_URI', + [default_params.find{ it.key == "SOURCE_REPO_URI"}?.value], + "Software repository URL (can not be modified)") stringParam("VERSION", default_params.find{ it.key == "VERSION"}?.value, "Packages version to be built or nightly (enable nightly build mode)") stringParam("RELEASE_VERSION", default_params.find{ it.key == "RELEASE_VERSION"}?.value, - "Packages release version") + "For downstream jobs: Packages release version") stringParam("RELEASE_REPO_BRANCH", default_params.find{ it.key == "RELEASE_REPO_BRANCH"}?.value, - "Branch from the -release repo to be used") + "For downstream jobs: Branch from the -release repo to be used") stringParam("UPLOAD_TO_REPO", default_params.find{ it.key == "UPLOAD_TO_REPO"}?.value, - "OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") + "For downstream jobs: OSRF repo name to upload the package to: stable | prerelease | nightly | none (for testing proposes)") } } } - static void create(Job job, Map default_params = [:]) + static void create(Job job, Map default_params = [:], Map default_hidden_params = [:]) { OSRFLinuxBuildPkgBase.create(job) GenericRemoteToken.create(job) OSRFSourceCreation.addParameters(job, default_params) + def pkg_sources_dir="pkgs" + job.with { - label Globals.nontest_label("docker") - wrappers { preBuildCleanup() } @@ -50,6 +55,9 @@ class OSRFSourceCreation priority 100 } + def canonical_package_name = Globals.get_canonical_package_name( + default_params.find{ it.key == "PACKAGE"}.value) + steps { systemGroovyCommand("""\ build.setDescription( @@ -62,14 +70,78 @@ class OSRFSourceCreation 'RTOOLS_BRANCH: ' + build.buildVariableResolver.resolve('RTOOLS_BRANCH')); """.stripIndent() ) + shell("""\ + #!/bin/bash -xe + + # Use Jammy/amd64 as base image to generate sources + export DISTRO=jammy + export ARCH=amd64 + /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash + """.stripIndent() + ) shell("""\ - #!/bin/bash -xe - export DISTRO=jammy - export ARCH=amd64 + #!/bin/bash -xe + + # Export information from the build in properties_files. The tarball extraction helps to + # deal with changes in the compression of the tarballs. + tarball=\$(find \${WORKSPACE}/${pkg_sources_dir} \ + -type f \ + -name ${canonical_package_name}-\${VERSION}.tar.* \ + -printf "%f\\n") + if [[ -z \${tarball} ]] || [[ \$(wc -w <<< \${tarball}) != 1 ]]; then + echo "Tarball name extraction returned \${tarball} which is not a one word string" + exit 1 + fi - /bin/bash -x ./scripts/jenkins-scripts/docker/gz-source-generation.bash - """.stripIndent()) + echo "TARBALL_NAME=\${tarball}" >> ${properties_file} + """.stripIndent() + ) + } + } + } + + // Useful to inject testing jobs + static void call_uploader_and_releasepy(Job job, + String repository_uploader_jobname, + String releasepy_jobname) + { + job.with + { + publishers { + postBuildScripts { + steps { + conditionalSteps { + condition { + not { + expression('none|None|^$','${ENV,var="UPLOAD_TO_REPO"}') + } + } + steps { + // Invoke repository_uploader + downstreamParameterized { + trigger(repository_uploader_jobname) { + parameters { + currentBuild() + predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: '${JOB_NAME}', + S3_UPLOAD_PATH: Globals.s3_upload_tarball_path(package_name)]) + propertiesFile(properties_file) // TARBALL_NAME + } + } + } + downstreamParameterized { + trigger(releasepy_jobname) { + parameters { + currentBuild() + predefinedProps([PROJECT_NAME_TO_COPY_ARTIFACTS: "\${JOB_NAME}"]) + propertiesFile(properties_file) // TARBALL_NAME + } + } + } + } + } + } + } } } } diff --git a/jenkins-scripts/dsl/test.dsl b/jenkins-scripts/dsl/test.dsl index c8f542e8d..08c9a991a 100644 --- a/jenkins-scripts/dsl/test.dsl +++ b/jenkins-scripts/dsl/test.dsl @@ -20,11 +20,63 @@ OSRFLinuxCompilationAnyGitHub.create(ignition_ci_pr_job, // releasing testing job def releasepy_job = job("_test_releasepy") OSRFReleasepy.create(releasepy_job, [DRY_RUN: true]) +releasepy_job.with { + blockOn("_test_repository_uploader") { + blockLevel('GLOBAL') + scanQueueFor('ALL') + } +} // gz source testing job def gz_source_job = job("_test_gz_source") OSRFSourceCreation.create(gz_source_job, [ - PACKAGE_NAME: "gz-cmake3", + PACKAGE: "gz-cmake3" , SOURCE_REPO_URI: "https://github.com/gazebosim/gz-cmake.git"]) +OSRFSourceCreation.call_uploader_and_releasepy(gz_source_job, + '_test_repository_uploader', + '_test_releasepy') +// repository_uploader fake test job + +def pkg_sources_dir = 'pkgs' +def repo_uploader = job("_test_repository_uploader") +OSRFBase.create(repo_uploader) +repo_uploader.with +{ + label Globals.nontest_label("docker") + + wrappers { + preBuildCleanup() + } + + parameters + { + stringParam('PACKAGE','','Package name') + stringParam('TARBALL_NAME', '', 'Tarball name to upload') + stringParam('S3_UPLOAD_PATH','', 'S3 path to upload') + stringParam('UPLOAD_TO_REPO','none','repo to upload') + } + + steps + { + copyArtifacts('_test_gz_source') + { + includePatterns("${pkg_sources_dir}/\${TARBALL_NAME}") + buildSelector { + upstreamBuild() + } + } + + shell("""\ + #!/bin/bash -xe + + # check that the tarball name actually exist + + ls -R \${WORKSPACE} + test -f \${WORKSPACE}/${pkg_sources_dir}/\${TARBALL_NAME} + + echo "Fake upload of \${TARBALL_NAME} to \${S3_UPLOAD_PATH}" + """.stripIndent()) + } +} // ------------------------------------------------------------------- def outdated_job_runner = job("_test_outdated_job_runner") From b4994d1f9109091a51467ea95da02fee7da63c48 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 26 Sep 2023 20:55:19 +0200 Subject: [PATCH 06/12] Include all gazebo releases in gz-collection.yaml (#1017) * Update gz-collections with all supported distros --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gz-collections.yaml | 253 ++++++++++++++++++++ jenkins-scripts/dsl/ignition_collection.dsl | 229 +++++++++--------- 2 files changed, 373 insertions(+), 109 deletions(-) diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index d06eab493..0eec17016 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -28,22 +28,234 @@ spec_version: 0 collections: - name: 'citadel' libs: + - name: ign-cmake + major_version: 2 + repo: + current_branch: ign-cmake2 + - name: ign-tools + major_version: 1 + repo: + current_branch: ign-tools1 + - name: ign-math + major_version: 6 + repo: + current_branch: ign-math6 + - name: ign-plugin + major_version: 1 + repo: + current_branch: ign-plugin1 + - name: ign-common + major_version: 3 + repo: + current_branch: ign-common3 + - name: ign-msgs + major_version: 5 + repo: + current_branch: ign-msgs5 + - name: ign-rendering + major_version: 3 + repo: + current_branch: ign-rendering3 + - name: sdformat + major_version: 9 + repo: + current_branch: sdf9 + - name: ign-fuel-tools + major_version: 4 + repo: + current_branch: ign-fuel-tools4 + - name: ign-transport + major_version: 8 + repo: + current_branch: ign-transport8 + - name: ign-gui + major_version: 3 + repo: + current_branch: ign-gui3 + - name: ign-sensors + major_version: 3 + repo: + current_branch: ign-sensors3 + - name: ign-physics + major_version: 2 + repo: + current_branch: ign-physics2 + - name: ign-gazebo + major_version: 7 + repo: + current_branch: ign-gazebo3 + - name: ign-launch + major_version: 2 + repo: + current_branch: ign-launch2 + - name: ign-citadel + major_version: 1 + repo: + current_branch: main ci: configs: - bionic packaging: + linux: + package_name: + ignore_major_version: + - ign-citadel - name: 'fortress' libs: + - name: gz-cmake + major_version: 3 + repo: + current_branch: gz-cmake2 + - name: gz-tools + major_version: 1 + repo: + current_branch: gz-tools1 + - name: gz-utils + major_version: 1 + repo: + current_branch: gz-utils1 + - name: gz-math + major_version: 6 + repo: + current_branch: gz-math6 + - name: gz-plugin + major_version: 1 + repo: + current_branch: gz-plugin1 + - name: gz-common + major_version: 4 + repo: + current_branch: gz-common4 + - name: gz-msgs + major_version: 8 + repo: + current_branch: gz-msgs8 + - name: gz-rendering + major_version: 6 + repo: + current_branch: gz-rendering6 + - name: sdformat + major_version: 12 + repo: + current_branch: sdf12 + - name: gz-fuel-tools + major_version: 7 + repo: + current_branch: gz-fuel-tools7 + - name: gz-transport + major_version: 11 + repo: + current_branch: gz-transport11 + - name: gz-gui + major_version: 6 + repo: + current_branch: gz-gui6 + - name: gz-sensors + major_version: 7 + repo: + current_branch: gz-sensors7 + - name: gz-physics + major_version: 6 + repo: + current_branch: gz-physics6 + - name: gz-sim + major_version: 6 + repo: + current_branch: gz-sim6 + - name: gz-launch + major_version: 5 + repo: + current_branch: gz-launch5 + - name: gz-fortress + major_version: 1 + repo: + current_branch: main ci: configs: - focal packaging: + linux: + package_name: + ignore_major_version: + - gz-fortress - name: 'garden' libs: + - name: gz-cmake + major_version: 3 + repo: + current_branch: gz-cmake3 + - name: gz-tools + major_version: 2 + repo: + current_branch: gz-tools2 + - name: gz-utils + major_version: 2 + repo: + current_branch: gz-utils2 + - name: gz-math + major_version: 7 + repo: + current_branch: gz-math7 + - name: gz-plugin + major_version: 2 + repo: + current_branch: gz-plugin2 + - name: gz-common + major_version: 5 + repo: + current_branch: gz-common5 + - name: gz-msgs + major_version: 9 + repo: + current_branch: gz-msgs9 + - name: gz-rendering + major_version: 7 + repo: + current_branch: gz-rendering7 + - name: sdformat + major_version: 13 + repo: + current_branch: sdf13 + - name: gz-fuel-tools + major_version: 8 + repo: + current_branch: gz-fuel-tools8 + - name: gz-transport + major_version: 12 + repo: + current_branch: gz-transport12 + - name: gz-gui + major_version: 7 + repo: + current_branch: gz-gui7 + - name: gz-sensors + major_version: 7 + repo: + current_branch: gz-sensors7 + - name: gz-physics + major_version: 6 + repo: + current_branch: gz-physics6 + - name: gz-sim + major_version: 7 + repo: + current_branch: gz-sim7 + - name: gz-launch + major_version: 6 + repo: + current_branch: gz-launch6 + - name: gz-garden + major_version: 1 + repo: + current_branch: main ci: configs: - focal packaging: + linux: + package_name: + ignore_major_version: + - gz-garden - name: 'harmonic' libs: - name: gz-cmake @@ -125,6 +337,46 @@ collections: nightly: distros: - jammy + - name: '__upcoming__' + libs: + - name: gz-cmake + major_version: 4 + repo: + current_branch: main + - name: gz-tools + major_version: 3 + repo: + current_branch: main + - name: gz-utils + major_version: 3 + repo: + current_branch: main + - name: gz-math + major_version: 8 + repo: + current_branch: main + - name: gz-plugin + major_version: 3 + repo: + current_branch: main + - name: gz-common + major_version: 6 + repo: + current_branch: main + - name: gz-msgs + major_version: 11 + repo: + current_branch: main + - name: gz-physics + major_version: 7 + repo: + current_branch: main + ci: + configs: + - jammy + packaging: + exclude: + - __upcoming__ ci_configs: - name: bionic system: @@ -183,6 +435,7 @@ ci_configs: - gz-sensors exclude: - gz-harmonic + - __upcoming__ pre_setup_script_hook: gz-physics: - "export MAKE_JOBS=1" diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index e140b089d..e30fcaff5 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -366,26 +366,83 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) gz_collections_yaml.collections.each { collection -> gz_collection_name = collection.name - // COLCON - Windows - def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") - Globals.gazebodistro_branch = true - OSRFWinCompilation.create(gz_win_ci_job, false) - gz_win_ci_job.with + dashboard_name = "ign-${gz_collection_name}" + if (gz_collection_name == "__upcoming__") + dashboard_name = gz_collection_name + + // Gazebo dashboards + // -------------------------------------------------------------- + dashboardView(dashboard_name) { + jobs { + gz_collection_jobs["${gz_collection_name}"].each { jobname -> + name(jobname) + } + if (collection.packaging?.linux?.nightly) { + collection.libs.each { lib -> + name(get_debbuilder_name(lib, collection.packaging)) + } + } + } + + columns { + status() + weather() + name() + testResult(0) + lastSuccess() + lastFailure() + lastDuration() + buildButton() + + } + + bottomPortlets { + jenkinsJobsList { + displayName('Jenkins jobs list') + } + } + + configure { view -> + view / columns << "hudson.plugins.warnings.WarningsColumn" (plugin: 'warnings@5.0.1') + + def topPortlets = view / NodeBuilder.newInstance().topPortlets {} + + topPortlets << 'hudson.plugins.view.dashboard.core.UnstableJobsPortlet' { + id createPortletId() + name 'Failing jobs' + showOnlyFailedJobs 'true' + recurse 'false' + } + } + } + + if (! collection.packaging.exclude?.contains(gz_collection_name)) { + // DEBBUILD: linux package builder + // -------------------------------------------------------------- + def build_pkg_job = job("gz-${gz_collection_name}-debbuilder") + OSRFLinuxBuildPkg.create(build_pkg_job) + build_pkg_job.with + { steps { - batchFile("""\ - set IGNITION_COLLECTION=${gz_collection_name} - call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" + shell("""\ + #!/bin/bash -xe + + /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash """.stripIndent()) } + } } - Globals.gazebodistro_branch = false + collection.ci.configs.each { ci_config_name -> ci_config = gz_collections_yaml.ci_configs.find { it.name == ci_config_name } distro = ci_config.system.version arch = ci_config.system.arch + if (ci_config.exclude.contains(gz_collection_name)) + return + // INSTALL JOBS: // -------------------------------------------------------------- if ((gz_collection_name == "citadel") || (gz_collection_name == "fortress")) { @@ -442,121 +499,75 @@ gz_collections_yaml.collections.each { collection -> """.stripIndent()) } } - } - // MAC Brew CI job - // -------------------------------------------------------------- - def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") - OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) - OSRFGitHub.create(gz_brew_ci_job, - "gazebosim/gz-${gz_collection_name}", - "main", - "ign-${gz_collection_name}") - gz_brew_ci_job.with - { - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -xe - "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" - """.stripIndent()) - } - } - - // MAC Brew bottle install job - // -------------------------------------------------------------- - def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") - OSRFBrewInstall.create(gz_brew_install_bottle_job) - - gz_brew_install_bottle_job.with - { - triggers { - cron('@daily') - } - - def bottle_name = "ignition-${gz_collection_name}" - - steps { - shell("""\ - #!/bin/bash -xe - - /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${bottle_name} - """.stripIndent()) - } - - publishers + // COLCON - Windows + Globals.gazebodistro_branch = true + def gz_win_ci_job = job("ign_${gz_collection_name}-ci-win") + OSRFWinCompilation.create(gz_win_ci_job, false) + gz_win_ci_job.with { - configure { project -> - project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { - unstableOnWarning true - failBuildOnError false - parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') - } - } + steps { + batchFile("""\ + set IGNITION_COLLECTION=${gz_collection_name} + call "./scripts/jenkins-scripts/lib/ign_collection-base.bat" + """.stripIndent()) + } } - } - - // DEBBUILD: linux package builder - // -------------------------------------------------------------- - def build_pkg_job = job("gz-${gz_collection_name}-debbuilder") - OSRFLinuxBuildPkg.create(build_pkg_job) - build_pkg_job.with - { - steps { - shell("""\ - #!/bin/bash -xe + Globals.gazebodistro_branch = false - /bin/bash -x ./scripts/jenkins-scripts/docker/multidistribution-ignition-debbuild.bash - """.stripIndent()) - } - } + // MAC Brew CI job + // -------------------------------------------------------------- + def gz_brew_ci_job = job("ignition_${gz_collection_name}-ci-main-homebrew-amd64") + OSRFBrewCompilation.create(gz_brew_ci_job, DISABLE_TESTS) + OSRFGitHub.create(gz_brew_ci_job, + "gazebosim/gz-${gz_collection_name}", + "main", + "ign-${gz_collection_name}") + gz_brew_ci_job.with + { + steps { + shell("""\ + #!/bin/bash -xe - // Gazebo dashboards - dashboardView("ign-${gz_collection_name}") - { - jobs { - gz_collection_jobs["${gz_collection_name}"].each { jobname -> - name(jobname) + /bin/bash -xe + "./scripts/jenkins-scripts/lib/project-default-devel-homebrew-amd64.bash" "gz-${gz_collection_name}" + """.stripIndent()) } - if (collection.packaging?.linux?.nightly) { - collection.libs.each { lib -> - name(get_debbuilder_name(lib, collection.packaging)) - } - } - } + } - columns { - status() - weather() - name() - testResult(0) - lastSuccess() - lastFailure() - lastDuration() - buildButton() + // MAC Brew bottle install job + // -------------------------------------------------------------- + def gz_brew_install_bottle_job = job("ignition_${gz_collection_name}-install_bottle-homebrew-amd64") + OSRFBrewInstall.create(gz_brew_install_bottle_job) + gz_brew_install_bottle_job.with + { + triggers { + cron('@daily') } - bottomPortlets { - jenkinsJobsList { - displayName('Jenkins jobs list') - } - } + def bottle_name = "ignition-${gz_collection_name}" - configure { view -> - view / columns << "hudson.plugins.warnings.WarningsColumn" (plugin: 'warnings@5.0.1') + steps { + shell("""\ + #!/bin/bash -xe - def topPortlets = view / NodeBuilder.newInstance().topPortlets {} + /bin/bash -x ./scripts/jenkins-scripts/lib/project-install-homebrew.bash ${bottle_name} + """.stripIndent()) + } - topPortlets << 'hudson.plugins.view.dashboard.core.UnstableJobsPortlet' { - id createPortletId() - name 'Failing jobs' - showOnlyFailedJobs 'true' - recurse 'false' + publishers + { + configure { project -> + project / publishers << 'hudson.plugins.logparser.LogParserPublisher' { + unstableOnWarning true + failBuildOnError false + parsingRulesPath('/var/lib/jenkins/logparser_warn_on_mark_unstable') + } } } - } + } + } } // NIGHTLY GENERATION From d0e2db33a99977e31bab3b03f974a4786f3af6ef Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Tue, 26 Sep 2023 15:06:04 -0700 Subject: [PATCH 07/12] brew_release.dsl: disable bigsur (#1018) homebrew-core has stopped building bigsur bottles, so we will as well. Signed-off-by: Steve Peters --- jenkins-scripts/dsl/brew_release.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/brew_release.dsl b/jenkins-scripts/dsl/brew_release.dsl index 2964076c8..e3e22a916 100644 --- a/jenkins-scripts/dsl/brew_release.dsl +++ b/jenkins-scripts/dsl/brew_release.dsl @@ -4,7 +4,7 @@ import javaposse.jobdsl.dsl.Job Globals.default_emails = "jrivero@osrfoundation.org, scpeters@osrfoundation.org" // first distro in list is used as touchstone -brew_supported_distros = [ "bigsur", "monterey", "ventura" ] +brew_supported_distros = [ "monterey", "ventura" ] bottle_hash_updater_job_name = 'generic-release-homebrew_pr_bottle_hash_updater' bottle_builder_job_name = 'generic-release-homebrew_triggered_bottle_builder' directory_for_bottles = 'pkgs' From 0f925822c5212f957fcc7931113fa136737e3417 Mon Sep 17 00:00:00 2001 From: Steve Peters Date: Mon, 2 Oct 2023 03:02:25 -0700 Subject: [PATCH 08/12] Fix CI for pull requests from forks (#1021) Signed-off-by: Steve Peters --- .github/workflows/ci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41c07b533..f0ee0cdd8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,8 +15,9 @@ jobs: - name: Idenfify files changed in this PR id: files run: | - git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }} - echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" >> $GITHUB_OUTPUT + git fetch origin ${{ github.ref }} + git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD + echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...FETCH_HEAD| tr '\n' ' ')" >> $GITHUB_OUTPUT - name: Run testing on changed config files id: dsl_check run: | From b7e7475918fe232acee73200ca0d1ae690859d69 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 12:02:43 +0200 Subject: [PATCH 09/12] Fix the check for existing github repository (#1019) Signed-off-by: Jose Luis Rivero --- release.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/release.py b/release.py index 521289c3d..6b42a21bd 100755 --- a/release.py +++ b/release.py @@ -183,12 +183,8 @@ def parse_args(argv): return args def get_release_repository_info(package): - # Do not use git@github method since it fails in non existant repositories - # asking for stdin user/pass. Same happen if no user/pass is provided - # using the fake foo:foo here seems to work - github_test_url = "https://foo:foo@github.com/gazebo-release/" + package + "-release" - if (github_repo_exists(github_test_url)): - github_url = "https://github.com/gazebo-release/" + package + "-release" + github_url = "https://github.com/gazebo-release/" + package + "-release" + if (github_repo_exists(github_url)): return 'git', github_url error("release repository not found in github.com/gazebo-release") From 948fbf16db1e6aee6f82a9e9285ee427d25c1008 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 18:44:30 +0200 Subject: [PATCH 10/12] Support for ROS 2 Iron CI in gzgarden wrappers (#960) Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ros_gz_bridge.dsl | 33 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index 245c5c671..80492d865 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -10,6 +10,11 @@ def bridge_packages = [ 'ros_gz_sim_demos' ] +def gzgarden_ros_distros_ci = [ + 'humble', + 'iron' +] + // BLOOM PACKAGE BUILDER JOBS bridge_packages.each { pkg -> pkg_dashed = pkg.replaceAll("_", "-") @@ -40,7 +45,7 @@ bridge_packages.each { pkg -> stringParam("LINUX_DISTRO", 'ubuntu', "Linux distribution to build packages for") stringParam("DISTRO", "jammy", "Linux release inside LINUX_DISTRO to build packages for") stringParam("ARCH", "amd64", "Architecture to build packages for") - stringParam('ROS_DISTRO', 'humble','ROS DISTRO to build pakcages for') + stringParam('ROS_DISTRO', null,'ROS DISTRO to build pakcages for') stringParam("UPLOAD_TO_REPO", 'stable', "OSRF repo name to upload the package to") stringParam('UPSTREAM_RELEASE_REPO', 'https://github.com/j-rivero/ros_ign-release', 'Release repository url') } @@ -191,18 +196,20 @@ manual_install_test_job.with } } -def periodic_install_test_job = job("ros_gzgarden_bridge-install-pkg_humble-ci-jammy-amd64") -generate_install_test_job(periodic_install_test_job) -periodic_install_test_job.with -{ - parameters { - stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") - stringParam("ARCH", 'amd64', "Architecture to build packages for") - stringParam("ROS_DISTRO", 'humble', "ROS distribution") - stringParam("GZ_VERSION", 'garden', "Gazebo version") - } +gzgarden_ros_distros_ci.each { ros_distro -> + def periodic_install_test_job = job("ros_gzgarden_bridge-install-pkg_${ros_distro}-ci-jammy-amd64") + generate_install_test_job(periodic_install_test_job) + periodic_install_test_job.with + { + parameters { + stringParam("DISTRO", 'jammy', "Linux release inside LINUX_DISTRO to build packages for") + stringParam("ARCH", 'amd64', "Architecture to build packages for") + stringParam("ROS_DISTRO", ros_distro, "ROS distribution") + stringParam("GZ_VERSION", 'garden', "Gazebo version") + } - triggers { - scm('@daily') + triggers { + scm('@daily') + } } } From bea701aa436bfb9031da16d2e9280369233c849a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 20:19:45 +0200 Subject: [PATCH 11/12] Patches and doc for GzHarmonic ros_gz creation (#1003) * Complete README with custom RELEASE_REPO_URL * Fix problems with renaming different Gazebo distros * Add the list of active releases --------- Signed-off-by: Jose Luis Rivero --- bloom/ros_gz/Dockerfile | 8 ++++---- bloom/ros_gz/README.md | 22 ++++++++++++++++++++++ bloom/ros_gz/rename-ros_gz-pkgs.bash | 4 +++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/bloom/ros_gz/Dockerfile b/bloom/ros_gz/Dockerfile index 620cd9931..628e39865 100644 --- a/bloom/ros_gz/Dockerfile +++ b/bloom/ros_gz/Dockerfile @@ -17,11 +17,11 @@ RUN apt-get update && apt-get install -y \ wget \ && rm -rf /var/lib/apt/lists/* -# Install osrf-rosdep for Gazebo distributions not in ROS +# Install osrf-rosdep for Gazebo distributions not in ROS. +# Note that 01 is important to give priority to the replace list below RUN sudo bash -c \ - 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/00-gazebo.list' -# Replace url -# since it does not support arbitrary environment variables + 'wget https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/00-gazebo.list -O /etc/ros/rosdep/sources.list.d/01-gazebo.list' +# Bloom does not support environment vars, use the trick or replacing one rosdep keys by others RUN sudo bash -c \ "wget $URL_OSRF_ROSDEP_REPLACE -O /etc/ros/rosdep/sources.list.d/00-replace-gz.list" diff --git a/bloom/ros_gz/README.md b/bloom/ros_gz/README.md index 466b17005..f0840eee8 100644 --- a/bloom/ros_gz/README.md +++ b/bloom/ros_gz/README.md @@ -27,6 +27,14 @@ Although using the officially supported version is the recommended way specially for non experienced users, some use cases might need to use a newer version of Gazebo than the one selected in REP-2000. +### List of active relases + +| Gazebo Release | ROS / ROS 2 Release | status | ros_gz branch | -release repository | +| ---------------|---------------------|------------|---------------|---------------------| +| Garden | Humble | stable | humble | https://github.com/gazebo-release/ros_ign-release | +| Garden | Iron | prerelease | iron | https://github.com/gazebo-release/ros_ign-release | +| Harmonic | Iron | prerelease | ros2 | https://github.com/j-rivero/ros_ign-gzharmonic-release | + ### Upstream versions released using this tutorial The `gbp -release repository` hosts the latest version released by the @@ -35,6 +43,7 @@ version the version of `ros_gz` released will be the latest one existing in the official `gbp -release repository`. The version would be the same but the release number will start on 1000. + ## 2. Initial setup To release a modified version of `ros_gz` which supports a different major @@ -55,6 +64,19 @@ package name. - Usage: `$ rename-ros_gz-pkgs.bash ` - Example: `$ rename-ros_gz-pkgs.bash garden humble` +The script supports to inject a custom RELEASE_REPO_URL that points to a bloom gbp +repository different than https://github.com/gazebo-release/ros_ign-release. + +``` +i.e use a https://github.com/gazebo-testing/ros_ign-gzharmonic-release as gbp testing repository + +RELEASE_REPO_URL=https://github.com/gazebo-testing/ros_ign-gzharmonic-release \ + ./bloom_from_special_env.bash \ + humble \ + garden \ + https://raw.githubusercontent.com/osrf/osrf-rosdep/master/gz/replace_fortress_with_garden/00-replace-gz-fortress-with-garden.list +``` + ### 2.2 Create a custom track in tracks.yml diff --git a/bloom/ros_gz/rename-ros_gz-pkgs.bash b/bloom/ros_gz/rename-ros_gz-pkgs.bash index 66b3a7953..6f02fe75c 100755 --- a/bloom/ros_gz/rename-ros_gz-pkgs.bash +++ b/bloom/ros_gz/rename-ros_gz-pkgs.bash @@ -47,7 +47,9 @@ for pkg in ${PKGS}; do git commit debian/control.em debian/changelog.em -m "Patch name to release ${GZ_RELEASE} version" # Include conflict with initial package name in ROS sed -i -e '/^Depends/a\Conflicts: \@(Package)' debian/control.em - + if [[ -n ${GZ_RELEASE_TO_CONFLICT} ]]; then + sed -i -e "s/Conflicts: @(Package)/Conflicts: @(Package.replace('-gz','-gz${GZ_RELEASE_TO_CONFLICT}'))/" debian/control.em + fi git commit debian/control.em -m "Set up a conflict with official ROS packages" git push origin "debian/$distro/$pkg" done From 983e8481577fa3c1ec3489f441c5c4004bdea6f6 Mon Sep 17 00:00:00 2001 From: "Addisu Z. Taddese" Date: Tue, 3 Oct 2023 06:07:52 -0500 Subject: [PATCH 12/12] Make prevous version optional in source_changelog.bash (#1012) The latest tagged version is used as the default. Signed-off-by: Addisu Z. Taddese Co-authored-by: Jose Luis Rivero --- source-repo-scripts/source_changelog.bash | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/source-repo-scripts/source_changelog.bash b/source-repo-scripts/source_changelog.bash index eb733dbb4..71a8aea23 100755 --- a/source-repo-scripts/source_changelog.bash +++ b/source-repo-scripts/source_changelog.bash @@ -1,10 +1,18 @@ #!/bin/bash +# Generates a list of changes since the last tagged version. +# bash source_changelog.bash +# +# Optionally, the previous version can be provided # bash source_changelog.bash - -PREV_VER=$1 +# +# E.g. +# bash source_changelog.bash 3.0.0 git fetch --tags +PREV_VER=${1:-$(git describe --tags --abbrev=0 | sed 's/.*_//')} +echo "Changes since $PREV_VER" + ORIGIN_URL=$(git remote get-url origin) REPO=$(basename ${ORIGIN_URL%.git}) @@ -14,7 +22,7 @@ PREV_TAG=$(git tag | grep "_${PREV_VER}$") # Compare current branch to PREV_TAG BRANCH=$(git rev-parse --abbrev-ref HEAD) -COMMITS=$(git log ${BRANCH}...${PREV_TAG} --pretty=format:"%h") +COMMITS=$(git log ${BRANCH}...${PREV_TAG} --no-merges --pretty=format:"%h") for COMMIT in $COMMITS do