From b09777157e56e348b28f3d1898145be57ceacd38 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Sep 2023 17:53:13 +0200 Subject: [PATCH 1/6] Enable push checks Signed-off-by: Jose Luis Rivero --- .github/workflows/ci.yaml | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41c07b533..7e7ccbb9b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: CI # base_ref / head_reaf are only available in PRs -on: [pull_request] +on: [pull_request, push] jobs: dsl_ci: @@ -12,39 +12,18 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Idenfify files changed in this PR - id: files - run: | - git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }} - echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" >> $GITHUB_OUTPUT - - name: Run testing on changed config files - id: dsl_check - run: | - for changed_file in ${{ steps.files.outputs.changed-files }}; do - if [[ ${changed_file} != ${changed_file/dsl\/*} ]]; then - echo "+ Detected at leat one config file: ${changed_file}." - echo "run_job=true" >> $GITHUB_OUTPUT - break - else - echo "run_job=false" >> $GITHUB_OUTPUT - fi - done - name: Checkout - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/checkout@v3 with: fetch-depth: 2 - uses: actions/setup-java@v3 - if: steps.dsl_check.outputs.run_job == 'true' with: distribution: 'temurin' java-version: '11' - name: Download and setup job dsl jar - if: steps.dsl_check.outputs.run_job == 'true' run: ./jenkins-scripts/dsl/tools/setup_local_generation.bash - name: Generate all DSL files - if: steps.dsl_check.outputs.run_job == 'true' run: | # simulate token for brew_release sudo mkdir -p /var/lib/jenkins/ && sudo touch /var/lib/jenkins/remote_token @@ -56,7 +35,6 @@ jobs: mv *.xml /tmp/pr_xml_configuration/ mv *.txt /tmp/pr_log_generated/ - name: Generate master DSL files - if: steps.dsl_check.outputs.run_job == 'true' run: | git clean -f -e jobdsl.jar git checkout master @@ -66,26 +44,22 @@ jobs: mv *.xml /tmp/current_xml_configuration/ mv *.txt /tmp/current_log_generated/ || true - name: Generating diffs - if: steps.dsl_check.outputs.run_job == 'true' run: | # somehow the Jenkins views changed the portlet_ id on every run. diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_files_changed.diff || true diff -ur -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_content_changed.diff || true diff -ur /tmp/current_log_generated /tmp/pr_log_generated > /tmp/log_content_changed.diff || true - name: Archive files changes - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: xml_config_files_changed path: /tmp/xml_config_files_changed.diff - name: Archive content changes - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: xml_config_content_changed path: /tmp/xml_config_content_changed.diff - name: Archive log changes - if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: log_content_changed From 7473a18f44cb9c43b2e108f9ba2f41553db9cf56 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Sep 2023 17:53:17 +0200 Subject: [PATCH 2/6] Just report tru on linux pr_any Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/ignition.dsl | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/jenkins-scripts/dsl/ignition.dsl b/jenkins-scripts/dsl/ignition.dsl index 566a601f4..45f4bafdd 100644 --- a/jenkins-scripts/dsl/ignition.dsl +++ b/jenkins-scripts/dsl/ignition.dsl @@ -443,31 +443,13 @@ gz_software.each { gz_sw -> def gz_ci_job_name = "ignition_${software_name}-ci-pr_any-ubuntu_auto-${arch}" def gz_ci_any_job = job(gz_ci_job_name) def gz_checkout_dir = "ign-${software_name}" - OSRFLinuxCompilationAnyGitHub.create(gz_ci_any_job, - "gazebosim/${gz_checkout_dir}", - enable_testing(software_name)) - include_gpu_label_if_needed(gz_ci_any_job, software_name) + GenericAnyJobGitHub.create(gz_ci_any_job, + "gazebosim/${gz_checkout_dir}", + [], + true) gz_ci_any_job.with { - if (gz_sw == 'physics') { - label Globals.nontest_label("large-memory") - extra_str += '\nexport MAKE_JOBS=1' - } - - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${ci_distro_str} - - ${GLOBAL_SHELL_CMD} - - export ARCH=${arch} - - /bin/bash -xe ./scripts/jenkins-scripts/docker/gz_${gz_sw.replaceAll('-','_')}-compilation.bash - """.stripIndent()) - } // end of steps + description 'Automatic generated job by DSL jenkins. Stub job for migration, not doing any check' } // end of ci_any_job // add ci-pr_any to the list for CIWorkflow From fbe24f3ccc74765e3d930e8d3188cbb70003c2ef Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Sep 2023 18:32:53 +0200 Subject: [PATCH 3/6] Geneate -ci-pr_any- for all jobs using yaml Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index ce6181ab4..492e2535e 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -75,10 +75,6 @@ boolean is_testing_enabled(lib_name, ci_config) void generate_ciconfigs_by_lib(config, configs_per_lib_index) { config.collections.each { collection -> - // TODO(jrivero): limit to harmonic for testing proposes - if (collection.name != 'harmonic') - return - collection.libs.each { lib -> def libName = lib.name def branch = lib.repo.current_branch @@ -181,6 +177,10 @@ configs_per_lib_index.each { lib_name, lib_configs -> // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) branches_with_collections.each { branch_and_collection -> + // TODO: remove after testing + if (branch_and_collection.collection == 'harmonic') + return + branch_name = branch_and_collection.branch def gz_ci_job = job("${gz_job_name_prefix}-ci-${branch_name}-${distro}-${arch}") generate_ci_job(gz_ci_job, lib_name, branch_name, ci_config) From 96730d4f1374b098a17dc5e207301941f93ff50a Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Sep 2023 18:43:16 +0200 Subject: [PATCH 4/6] Limit to Harmonic the -ci- branches testing Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/gazebo_libs.dsl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 492e2535e..cff29f441 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -178,7 +178,7 @@ configs_per_lib_index.each { lib_name, lib_configs -> // CI branch jobs (-ci-$branch-) (pulling check every 5 minutes) branches_with_collections.each { branch_and_collection -> // TODO: remove after testing - if (branch_and_collection.collection == 'harmonic') + if (branch_and_collection.collection != 'harmonic') return branch_name = branch_and_collection.branch From ecefecd9c7ca56f759dadced86951837b4d3af97 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 12:08:03 +0200 Subject: [PATCH 5/6] Revert "Enable push checks" This reverts commit b09777157e56e348b28f3d1898145be57ceacd38. --- .github/workflows/ci.yaml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7e7ccbb9b..41c07b533 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: CI # base_ref / head_reaf are only available in PRs -on: [pull_request, push] +on: [pull_request] jobs: dsl_ci: @@ -12,18 +12,39 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Idenfify files changed in this PR + id: files + run: | + git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }} + echo "changed-files=$(git diff --name-only origin/${{ github.base_ref }}...origin/${{ github.head_ref }}| tr '\n' ' ')" >> $GITHUB_OUTPUT + - name: Run testing on changed config files + id: dsl_check + run: | + for changed_file in ${{ steps.files.outputs.changed-files }}; do + if [[ ${changed_file} != ${changed_file/dsl\/*} ]]; then + echo "+ Detected at leat one config file: ${changed_file}." + echo "run_job=true" >> $GITHUB_OUTPUT + break + else + echo "run_job=false" >> $GITHUB_OUTPUT + fi + done - name: Checkout + if: steps.dsl_check.outputs.run_job == 'true' uses: actions/checkout@v3 with: fetch-depth: 2 - uses: actions/setup-java@v3 + if: steps.dsl_check.outputs.run_job == 'true' with: distribution: 'temurin' java-version: '11' - name: Download and setup job dsl jar + if: steps.dsl_check.outputs.run_job == 'true' run: ./jenkins-scripts/dsl/tools/setup_local_generation.bash - name: Generate all DSL files + if: steps.dsl_check.outputs.run_job == 'true' run: | # simulate token for brew_release sudo mkdir -p /var/lib/jenkins/ && sudo touch /var/lib/jenkins/remote_token @@ -35,6 +56,7 @@ jobs: mv *.xml /tmp/pr_xml_configuration/ mv *.txt /tmp/pr_log_generated/ - name: Generate master DSL files + if: steps.dsl_check.outputs.run_job == 'true' run: | git clean -f -e jobdsl.jar git checkout master @@ -44,22 +66,26 @@ jobs: mv *.xml /tmp/current_xml_configuration/ mv *.txt /tmp/current_log_generated/ || true - name: Generating diffs + if: steps.dsl_check.outputs.run_job == 'true' run: | # somehow the Jenkins views changed the portlet_ id on every run. diff -qr -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_files_changed.diff || true diff -ur -I '.*dashboard_portlet_.*.*' /tmp/current_xml_configuration /tmp/pr_xml_configuration > /tmp/xml_config_content_changed.diff || true diff -ur /tmp/current_log_generated /tmp/pr_log_generated > /tmp/log_content_changed.diff || true - name: Archive files changes + if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: xml_config_files_changed path: /tmp/xml_config_files_changed.diff - name: Archive content changes + if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: xml_config_content_changed path: /tmp/xml_config_content_changed.diff - name: Archive log changes + if: steps.dsl_check.outputs.run_job == 'true' uses: actions/upload-artifact@v3 with: name: log_content_changed From 9f8633a657b61c580832dd2aa123c39a9358cac7 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 2 Oct 2023 12:41:10 +0200 Subject: [PATCH 6/6] Generate the fake sdformat job for ubuntu-auto from sdformat.dsl Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/sdformat.dsl | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/jenkins-scripts/dsl/sdformat.dsl b/jenkins-scripts/dsl/sdformat.dsl index 27fa20683..d983d18f5 100644 --- a/jenkins-scripts/dsl/sdformat.dsl +++ b/jenkins-scripts/dsl/sdformat.dsl @@ -87,24 +87,12 @@ ci_distro.each { distro -> } // -------------------------------------------------------------- - // 2. Create the any job + // 2. Create a fake any job for migrating + // TODO: remove the job once the migration in https://github.com/gazebo-tooling/release-tools/issues/1010 + // is done. String sdf_repo = "gazebosim/sdformat" def sdformat_ci_any_job = job(ci_build_any_job_name_linux) - OSRFLinuxCompilationAnyGitHub.create(sdformat_ci_any_job, sdf_repo) - sdformat_ci_any_job.with - { - steps - { - shell("""\ - #!/bin/bash -xe - - export DISTRO=${ci_distro_str} - - export ARCH=${arch} - /bin/bash -xe ./scripts/jenkins-scripts/docker/sdformat-compilation.bash - """.stripIndent()) - } - } + GenericAnyJobGitHub.create(sdformat_ci_any_job, sdf_repo) } // end of arch } // end of distro