Improve ccache use on github workflows (#1243)

* Update merge group job * Move pr to pull_request for consistency * Define Celeritas hostname in github CI to improve caching * Add ccache debug and increase sizes * Add weekly cron job and ccache rebuild
celeritas-project · May 27, 2024 · 3d6f3c5 · 3d6f3c5
1 parent 08f509f
commit 3d6f3c5
Show file tree

Hide file tree

Showing 7 changed files with 102 additions and 9 deletions.
diff --git a/.github/workflows/build-ultralite.yml b/.github/workflows/build-ultralite.yml
@@ -8,12 +8,16 @@ concurrency:
   group: build-ultralite-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}-${{github.workflow}}
   cancel-in-progress: true
 
+# env:
+#   CCACHE_DEBUG: 1
+#   CCACHE_DEBUGDIR: ${{github.workspace}}/ccache-debug
+
 jobs:
   linux:
     name: ultralite-ubuntu
     env:
       CCACHE_DIR: "${{github.workspace}}/.ccache"
-      CCACHE_MAXSIZE: "20Mi"
+      CCACHE_MAXSIZE: "50Mi"
       CMAKE_PRESET: ultralite
     runs-on: ubuntu-latest
     steps:
@@ -61,11 +65,19 @@ jobs:
         if: ${{!cancelled()}}
         run: |
           ccache -s
+      - name: Upload ccache debug
+        if: ${{env.CCACHE_DEBUG && !cancelled()}}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{env.CCACHE_DEBUGDIR}}
+          name: ccache-debug-${{github.job}}
+          overwrite: true
+          retention-days: 7
   windows:
     name: ultralite-windows
     env:
       CCACHE_DIR: "${{github.workspace}}\\.ccache"
-      CCACHE_MAXSIZE: "20Mi"
+      CCACHE_MAXSIZE: "50Mi"
       CMAKE_PRESET: ultralite
     runs-on: windows-latest
     steps:
@@ -105,5 +117,13 @@ jobs:
         if: ${{!cancelled()}}
         run: |
           ccache -s
+      - name: Upload ccache debug
+        if: ${{env.CCACHE_DEBUG && !cancelled()}}
+        uses: actions/upload-artifact@v4
+        with:
+          path: ${{env.CCACHE_DEBUGDIR}}
+          name: ccache-debug-${{github.job}}
+          overwrite: true
+          retention-days: 7
 
 # vim: set nowrap tw=100:
diff --git a/.github/workflows/clear-cache.yml b/.github/workflows/clear-cache.yml
@@ -0,0 +1,29 @@
+name: clear-cache
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  clear-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clear ccache
+        uses: actions/github-script@v7
+        with:
+          script: |
+            console.log("Getting action cache")
+            const caches = await github.rest.actions.getActionsCacheList({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            })
+            for (const cache of caches.data.actions_caches) {
+              if (cache.key.startsWith("ccache-") {
+                console.log("Deleting cache entry", cache)
+                github.rest.actions.deleteActionsCacheById({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  cache_id: cache.id,
+                })
+              }
+            }
diff --git a/.github/workflows/cron_weekly.yml b/.github/workflows/cron_weekly.yml
@@ -0,0 +1,36 @@
+name: pull_request
+run-name: >-
+  ${{github.event.pull_request.title}}
+  (#${{github.event.number}})
+
+on:
+  schedule:
+    - cron: "33 06 * * 0"   # Sunday at 6:33 am UTC
+
+jobs:
+  clear-ccache:
+    uses: ./.github/workflows/clear-cache.yml
+  build-fast:
+    needs: [clear-cache]
+    uses: ./.github/workflows/build-fast.yml
+  build-ultralite:
+    needs: [clear-cache]
+    uses: ./.github/workflows/build-ultralite.yml
+  build-spack:
+    needs: [clear-cache]
+    uses: ./.github/workflows/build-spack.yml
+
+  all:
+    if: ${{always()}}
+    needs:
+    - build-fast
+    - build-spack
+    - build-ultralite
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check that all jobs succeeded
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{toJSON(needs)}}
+
+# vim: set nowrap tw=100:
diff --git a/.github/workflows/merge_group.yml b/.github/workflows/merge_group.yml
@@ -13,14 +13,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # TODO: use a more limited build for merge group? (or for PR?)
-  build:
-    uses: ./.github/workflows/build-full.yml
+  build-docker:
+    uses: ./.github/workflows/build-docker.yml
+  build-spack:
+    uses: ./.github/workflows/build-spack.yml
   all:
-    needs: [build]
+    if: ${{always()}}
+    needs:
+    - build-docker
+    - build-spack
     runs-on: ubuntu-latest
     steps:
-    - name: Success
-      run: "true"
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{toJSON(needs)}}
 
 # vim: set nowrap tw=100:
diff --git a/.github/workflows/pr.yml → .github/workflows/pull_request.yml b/.github/workflows/pr.yml → .github/workflows/pull_request.yml
@@ -1,4 +1,4 @@
-name: pr
+name: pull_request
 run-name: >-
   ${{github.event.pull_request.title}}
   (#${{github.event.number}})

diff --git a/scripts/cmake-presets/ci-ubuntu-github.json b/scripts/cmake-presets/ci-ubuntu-github.json
@@ -25,6 +25,7 @@
         "CELERITAS_USE_ROOT":    {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_SWIG":    {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"},
+        "CELERITAS_HOSTNAME": "ubuntu-github",
         "CMAKE_BUILD_TYPE":      "Release",
         "CMAKE_CXX_COMPILER_LAUNCHER": "ccache",
         "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},

diff --git a/scripts/cmake-presets/ci-windows-github.json b/scripts/cmake-presets/ci-windows-github.json
@@ -25,6 +25,7 @@
         "CELERITAS_USE_ROOT":    {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_SWIG":    {"type": "BOOL", "value": "OFF"},
         "CELERITAS_USE_VecGeom": {"type": "BOOL", "value": "OFF"},
+        "CELERITAS_HOSTNAME": "windows-github",
         "CMAKE_BUILD_TYPE":      "Release",
         "CMAKE_CXX_COMPILER_LAUNCHER": "ccache",
         "CMAKE_CXX_EXTENSIONS": {"type": "BOOL", "value": "OFF"},