From 3d6f3c539aa03b9ab0d60c0a45e3126cbda814f4 Mon Sep 17 00:00:00 2001 From: "Seth R. Johnson" Date: Mon, 27 May 2024 08:21:48 -0400 Subject: [PATCH] Improve ccache use on github workflows (#1243) * Update merge group job * Move pr to pull_request for consistency * Define Celeritas hostname in github CI to improve caching * Add ccache debug and increase sizes * Add weekly cron job and ccache rebuild --- .github/workflows/build-ultralite.yml | 24 +++++++++++-- .github/workflows/clear-cache.yml | 29 +++++++++++++++ .github/workflows/cron_weekly.yml | 36 +++++++++++++++++++ .github/workflows/merge_group.yml | 18 ++++++---- .../workflows/{pr.yml => pull_request.yml} | 2 +- scripts/cmake-presets/ci-ubuntu-github.json | 1 + scripts/cmake-presets/ci-windows-github.json | 1 + 7 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/clear-cache.yml create mode 100644 .github/workflows/cron_weekly.yml rename .github/workflows/{pr.yml => pull_request.yml} (98%) diff --git a/.github/workflows/build-ultralite.yml b/.github/workflows/build-ultralite.yml index e662dbc3f3..49952ab521 100644 --- a/.github/workflows/build-ultralite.yml +++ b/.github/workflows/build-ultralite.yml @@ -8,12 +8,16 @@ concurrency: group: build-ultralite-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}-${{github.workflow}} cancel-in-progress: true +# env: +# CCACHE_DEBUG: 1 +# CCACHE_DEBUGDIR: ${{github.workspace}}/ccache-debug + jobs: linux: name: ultralite-ubuntu env: CCACHE_DIR: "${{github.workspace}}/.ccache" - CCACHE_MAXSIZE: "20Mi" + CCACHE_MAXSIZE: "50Mi" CMAKE_PRESET: ultralite runs-on: ubuntu-latest steps: @@ -61,11 +65,19 @@ jobs: if: ${{!cancelled()}} run: | ccache -s + - name: Upload ccache debug + if: ${{env.CCACHE_DEBUG && !cancelled()}} + uses: actions/upload-artifact@v4 + with: + path: ${{env.CCACHE_DEBUGDIR}} + name: ccache-debug-${{github.job}} + overwrite: true + retention-days: 7 windows: name: ultralite-windows env: CCACHE_DIR: "${{github.workspace}}\\.ccache" - CCACHE_MAXSIZE: "20Mi" + CCACHE_MAXSIZE: "50Mi" CMAKE_PRESET: ultralite runs-on: windows-latest steps: @@ -105,5 +117,13 @@ jobs: if: ${{!cancelled()}} run: | ccache -s + - name: Upload ccache debug + if: ${{env.CCACHE_DEBUG && !cancelled()}} + uses: actions/upload-artifact@v4 + with: + path: ${{env.CCACHE_DEBUGDIR}} + name: ccache-debug-${{github.job}} + overwrite: true + retention-days: 7 # vim: set nowrap tw=100: diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml new file mode 100644 index 0000000000..939d2bcf79 --- /dev/null +++ b/.github/workflows/clear-cache.yml @@ -0,0 +1,29 @@ +name: clear-cache + +on: + workflow_dispatch: + workflow_call: + +jobs: + clear-cache: + runs-on: ubuntu-latest + steps: + - name: Clear ccache + uses: actions/github-script@v7 + with: + script: | + console.log("Getting action cache") + const caches = await github.rest.actions.getActionsCacheList({ + owner: context.repo.owner, + repo: context.repo.repo, + }) + for (const cache of caches.data.actions_caches) { + if (cache.key.startsWith("ccache-") { + console.log("Deleting cache entry", cache) + github.rest.actions.deleteActionsCacheById({ + owner: context.repo.owner, + repo: context.repo.repo, + cache_id: cache.id, + }) + } + } diff --git a/.github/workflows/cron_weekly.yml b/.github/workflows/cron_weekly.yml new file mode 100644 index 0000000000..045fddb4b5 --- /dev/null +++ b/.github/workflows/cron_weekly.yml @@ -0,0 +1,36 @@ +name: pull_request +run-name: >- + ${{github.event.pull_request.title}} + (#${{github.event.number}}) + +on: + schedule: + - cron: "33 06 * * 0" # Sunday at 6:33 am UTC + +jobs: + clear-ccache: + uses: ./.github/workflows/clear-cache.yml + build-fast: + needs: [clear-cache] + uses: ./.github/workflows/build-fast.yml + build-ultralite: + needs: [clear-cache] + uses: ./.github/workflows/build-ultralite.yml + build-spack: + needs: [clear-cache] + uses: ./.github/workflows/build-spack.yml + + all: + if: ${{always()}} + needs: + - build-fast + - build-spack + - build-ultralite + runs-on: ubuntu-latest + steps: + - name: Check that all jobs succeeded + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{toJSON(needs)}} + +# vim: set nowrap tw=100: diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml index ee028acf31..02601e0629 100644 --- a/.github/workflows/merge_group.yml +++ b/.github/workflows/merge_group.yml @@ -13,14 +13,20 @@ concurrency: cancel-in-progress: true jobs: - # TODO: use a more limited build for merge group? (or for PR?) - build: - uses: ./.github/workflows/build-full.yml + build-docker: + uses: ./.github/workflows/build-docker.yml + build-spack: + uses: ./.github/workflows/build-spack.yml all: - needs: [build] + if: ${{always()}} + needs: + - build-docker + - build-spack runs-on: ubuntu-latest steps: - - name: Success - run: "true" + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{toJSON(needs)}} # vim: set nowrap tw=100: diff --git a/.github/workflows/pr.yml b/.github/workflows/pull_request.yml similarity index 98% rename from .github/workflows/pr.yml rename to .github/workflows/pull_request.yml index 0c60832941..1f384e2ba1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pull_request.yml @@ -1,4 +1,4 @@ -name: pr +name: pull_request run-name: >- ${{github.event.pull_request.title}} (#${{github.event.number}}) diff --git a/scripts/cmake-presets/ci-ubuntu-github.json b/scripts/cmake-presets/ci-ubuntu-github.json index 49e1cde457..46633bcf93 100644 --- a/scripts/cmake-presets/ci-ubuntu-github.json +++ b/scripts/cmake-presets/ci-ubuntu-github.json @@ -25,6 +25,7 @@ "CELERITAS_USE_ROOT": {"type": "BOOL", "value": "OFF"}, "CELERITAS_USE_SWIG": {"type": "BOOL", "value": "OFF"}, "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}, + "CELERITAS_HOSTNAME": "ubuntu-github", "CMAKE_BUILD_TYPE": "Release", "CMAKE_CXX_COMPILER_LAUNCHER": "ccache", "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"}, diff --git a/scripts/cmake-presets/ci-windows-github.json b/scripts/cmake-presets/ci-windows-github.json index 0b05410583..4c5a504e3a 100644 --- a/scripts/cmake-presets/ci-windows-github.json +++ b/scripts/cmake-presets/ci-windows-github.json @@ -25,6 +25,7 @@ "CELERITAS_USE_ROOT": {"type": "BOOL", "value": "OFF"}, "CELERITAS_USE_SWIG": {"type": "BOOL", "value": "OFF"}, "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"}, + "CELERITAS_HOSTNAME": "windows-github", "CMAKE_BUILD_TYPE": "Release", "CMAKE_CXX_COMPILER_LAUNCHER": "ccache", "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},