separate data caches before combining

spacetelescope · Oct 4, 2023 · ca8de2f · ca8de2f
1 parent 1548341
commit ca8de2f
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 15 deletions.
diff --git a/.github/workflows/data.yml b/.github/workflows/data.yml
@@ -5,16 +5,16 @@ on:
     outputs:
       path:
         value: ${{ jobs.path.outputs.path }}
+      cache_key:
+        value: ${{ jobs.cache.outputs.key }}
       crds_path:
         value: ${{ jobs.crds.outputs.path }}
       crds_server:
         value: ${{ jobs.crds.outputs.server }}
       crds_context:
         value: ${{ jobs.crds.outputs.context }}
       webbpsf_path:
-        value: ${{ jobs.retrieve_webbpsf_data_hash.outputs.path }}
-      webbpsf_hash:
-        value: ${{ jobs.retrieve_webbpsf_data_hash.outputs.hash }}
+        value: ${{ jobs.path.outputs.path }}/webbpsf-data
   workflow_dispatch:
   schedule:
     - cron: "42 4 * * 3"
@@ -67,32 +67,41 @@ jobs:
         name: check downloaded data against the existing cache
         uses: actions/cache@v3
         with:
-          path: ${{ needs.path.outputs.path }}
-          key: data-${{ steps.download.outputs.hash }}-
+          path: ${{ needs.path.outputs.path }}/webbpsf-data
+          key: webbpsf-${{ steps.download.outputs.hash }}
       - if: ${{ steps.cache_download.outputs.cache-hit != 'true' }}
         name: extract data to cache directory
         run: |
           mkdir -p ${{ needs.path.outputs.path }}
           tar -xzvf tmp/webbpsf-data.tar.gz -C ${{ needs.path.outputs.path }}
-  retrieve_webbpsf_data_hash:
+  cache:
     needs: [ path, download_webbpsf_data ]
     # run regardless if `download_webbpsf_data' succeeds or is skipped
     if: always() && (needs.download_webbpsf_data.result == 'success' || needs.download_webbpsf_data.result == 'skipped')
-    name: retrieve hash of cached WebbPSF data
+    name: combine individual data caches
     runs-on: ubuntu-latest
     env:
       GH_TOKEN: ${{ github.token }}
     outputs:
-      path: ${{ needs.path.outputs.path }}/webbpsf-data
-      hash: ${{ steps.retrieve_hash.outputs.hash }}
+      key: ${{ steps.cache_key.outputs.key }}
     steps:
-      - id: retrieve_hash
+      - id: retrieve_webbpsf_hash
         name: retrieve data hash of latest cache key
         run: |
           # use actions/gh-actions-cache to allow filtering by key
           gh extension install actions/gh-actions-cache
-          LATEST_CACHE=$(gh actions-cache list -R spacetelescope/romancal --key data- --sort created-at | cut -f 1 | head -n 1)
+          LATEST_CACHE=$(gh actions-cache list -R spacetelescope/romancal --key webbpsf- --sort created-at | cut -f 1 | head -n 1)
           echo "LATEST_CACHE=$LATEST_CACHE"
           HASH=$(echo $LATEST_CACHE | cut -d '-' -f 2)
           echo "hash=$HASH" >> $GITHUB_OUTPUT
           if [ "$HASH" == '' ]; then exit 1; fi
+      - uses: actions/cache@v3
+        with:
+          path: ${{ needs.path.outputs.path }}/webbpsf-data
+          key: webbpsf-${{ steps.retrieve_webbpsf_hash.outputs.hash }}
+      - id: cache_key
+        run: echo "key=data-${{ steps.retrieve_webbpsf_hash.outputs.hash }}-" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v3
+        with:
+          path: ${{ needs.path.outputs.path }}
+          key: ${{ steps.cache_key.outputs.key }}
diff --git a/.github/workflows/roman_ci.yml b/.github/workflows/roman_ci.yml
@@ -42,8 +42,8 @@ jobs:
         DD_GIT_COMMIT_SHA: ${{ github.sha }}
         DD_GIT_BRANCH: ${{ github.ref_name }}
       cache-path: ${{ needs.data.outputs.path }}
-      cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }}
-      cache-restore-keys: data-${{ needs.data.outputs.webbpsf_hash }}-
+      cache-key: data-${{ needs.data.outputs.data_hash }}-${{ needs.data.outputs.crds_context }}
+      cache-restore-keys: ${{ needs.data.outputs.cache_key }}
       envs: |
         - linux: py39-oldestdeps-cov
           pytest-results-summary: true

diff --git a/.github/workflows/roman_ci_cron.yaml b/.github/workflows/roman_ci_cron.yaml
@@ -38,7 +38,7 @@ jobs:
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       cache-path: ${{ needs.data.outputs.crds_path }}
       cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }}
-      cache-restore-keys: data-${{ needs.data.outputs.webbpsf_hash }}-
+      cache-restore-keys: ${{ needs.data.outputs.cache_key }}
       envs: |
         - macos: py39
           pytest-results-summary: true

diff --git a/.github/workflows/tests_devdeps.yml b/.github/workflows/tests_devdeps.yml
@@ -40,7 +40,7 @@ jobs:
         CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
       cache-path: ${{ needs.data.outputs.crds_path }}
       cache-key: data-${{ needs.data.outputs.webbpsf_hash }}-${{ needs.data.outputs.crds_context }}
-      cache-restore-keys: data-${{ needs.data.outputs.webbpsf_hash }}-
+      cache-restore-keys: ${{ needs.data.outputs.cache_key }}
       envs: |
         - linux: py39-devdeps
         - macos: py39-devdeps