cache docker images

ethpandaops · Jan 12, 2025 · c0ff84f · c0ff84f
1 parent f4ceb31
commit c0ff84f
Show file tree

Hide file tree

Showing 5 changed files with 82 additions and 7 deletions.
diff --git a/.github/workflows/_shared-prepare-docker-cache.yaml b/.github/workflows/_shared-prepare-docker-cache.yaml
@@ -0,0 +1,47 @@
+
+on:
+  workflow_call:
+    outputs:
+      kurtosis_version:
+        description: "The latest version of the kurtosis cli (used as caching key)."
+        value: ${{ jobs.run_prepare.outputs.kurtosis_version }}
+
+jobs:
+  run_prepare:
+    name: "Prepare Docker cache"
+    runs-on: ubuntu-latest
+    outputs:
+      kurtosis_version: ${{ steps.kurtosis_version.outputs.kurtosis_version }}
+    steps:
+    - name: "Get kurtosis version"
+      id: kurtosis_version
+      shell: bash
+      run: |
+        kurtosis_version=$(curl -s https://api.github.com/repos/kurtosis-tech/kurtosis/releases/latest | jq -r .tag_name)
+        echo "kurtosis_version=$(echo "$kurtosis_version")" >> $GITHUB_OUTPUT
+    - name: Cache Docker images.
+      id: cache
+      uses: ScribeMD/[email protected]
+      with:
+        key: docker-${{ runner.os }}-${{ steps.kurtosis_version.outputs.kurtosis_version }}
+
+    - name: "Generate dummy kurtosis config"
+      shell: bash
+      run: |
+        mkdir -p ./temp
+        echo "{\"participants\": []}" > ./temp/test-network.yaml
+
+    - name: Run kurtosis testnet for cache warmup
+      if: ${{ steps.cache.outputs.cache-hit == 'false' }}
+      continue-on-error: true
+      id: testnet
+      uses: ethpandaops/kurtosis-assertoor-github-action@v1
+      with:
+        kurtosis_extra_args: "--image-download always --non-blocking-tasks --verbosity DETAILED"
+        kurtosis_backend: docker
+        ethereum_package_branch: main
+        ethereum_package_args: "./temp/test-network.yaml"
+        enclave_name: "assertoor-warmup"
+        await_assertoor_tests: "false"
+        enclave_dump: "false"
+
diff --git a/.github/workflows/_shared-run-test.yaml b/.github/workflows/_shared-run-test.yaml
@@ -82,6 +82,7 @@ jobs:
       uses: ScribeMD/[email protected]
       with:
         key: docker-${{ runner.os }}-${{ inputs.kurtosis_version }}
+        read-only: true
     - name: "Install shell dependencies"
       shell: bash
       run: |

diff --git a/.github/workflows/_shared-run.yaml b/.github/workflows/_shared-run.yaml
@@ -14,6 +14,10 @@ on:
         type: string
         description: "Use ChatGPT to generate a summary of failed tests (default: false)."
         default: "false"
+      kurtosis_version:
+        type: string
+        description: "The latest version of the kurtosis cli (used as caching key)."
+        default: ""
     secrets:
       RANCHER_TOKEN:
         description: "The rancher token to login to the rancher server."
@@ -34,7 +38,6 @@ jobs:
       name: ${{ steps.config.outputs.name }}
       clients: ${{ steps.config.outputs.clients }}
       kurtosis: ${{ steps.config.outputs.kurtosis }}
-      kurtosis_version: ${{ steps.config.outputs.kurtosis_version }}
       kurtosis_branch: ${{ steps.config.outputs.kurtosis_branch }}
       worker: ${{ steps.config.outputs.worker }}
       backend: ${{ steps.config.outputs.backend }}
@@ -59,7 +62,6 @@ jobs:
         name="$(echo "$testcfg" | jq -r .name)"
         clients="$(echo "$testcfg" | jq -r .clients)"
         kurtosis="$(echo "$testcfg" | jq -r .kurtosis)"
-        kurtosis_version=$(curl -s https://api.github.com/repos/kurtosis-tech/kurtosis/releases/latest | jq -r .tag_name)
         kurtosis_branch="$(echo "$testcfg" | jq -r '.kurtosis_branch // ""')"
         worker="$(echo "$testcfg" | jq -c .worker)"
         backend="$(echo "$testcfg" | jq -r .backend)"
@@ -76,8 +78,6 @@ jobs:
         echo "clients=$(echo "$clients")" >> $GITHUB_OUTPUT
         echo "Kurtosis yaml: $kurtosis"
         echo "kurtosis=$(echo "$kurtosis")" >> $GITHUB_OUTPUT
-        echo "Kurtosis version: $kurtosis_version"
-        echo "kurtosis_version=$(echo "$kurtosis_version")" >> $GITHUB_OUTPUT
         echo "Kurtosis package branch: $kurtosis_branch"
         echo "kurtosis_branch=$(echo "$kurtosis_branch")" >> $GITHUB_OUTPUT
         echo "Worker: $worker"
@@ -141,7 +141,7 @@ jobs:
       pairs: ${{ matrix.pairs }}
       worker: ${{ needs.get_test.outputs.worker }}
       kurtosis: ${{ needs.get_test.outputs.kurtosis }}
-      kurtosis_version: ${{ needs.get_test.outputs.kurtosis_version }}
+      kurtosis_version: ${{ inputs.kurtosis_version }}
       kurtosis_branch: ${{ needs.get_test.outputs.kurtosis_branch }}
       backend: ${{ needs.get_test.outputs.backend }}
       kubeCluster: ${{ needs.get_test.outputs.kubernetes_cluster }}

diff --git a/.github/workflows/run-manual.yml b/.github/workflows/run-manual.yml
@@ -26,6 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       test_configs: ${{ steps.tests.outputs.test_configs }}
+      has_docker_tests: ${{ steps.tests.outputs.has_docker_tests }}
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v4
@@ -69,13 +70,25 @@ jobs:
           test_configs="$(echo "$test_configs" | jq -c "(.[]).kurtosis_branch = \"$override_kurtosis_branch\"")"
         fi
 
+        has_docker_tests="false"
+        if [ $(echo "$test_configs" | jq -c ".[] | select(.backend == \"docker\")" | wc -l) -gt 0 ]; then
+          has_docker_tests="true"
+        fi
+
         echo "test_configs<<EOF" >> $GITHUB_OUTPUT
         echo "$test_configs" >> $GITHUB_OUTPUT
         echo "$(echo "$test_configs" | jq)"
         echo "EOF" >> $GITHUB_OUTPUT
 
-  run_tests:
+        echo "has_docker_tests=$has_docker_tests" >> $GITHUB_OUTPUT
+
+  prepare_cache:
     needs: get_tests
+    if: "${{ needs.get_tests.outputs.has_docker_tests == 'true' }}"
+    uses: ./.github/workflows/_shared-prepare-docker-cache.yaml
+
+  run_tests:
+    needs: [get_tests, prepare_cache]
     uses: ./.github/workflows/_shared-run.yaml
     name: "${{ matrix.config.name }}"
     strategy:
@@ -85,6 +98,7 @@ jobs:
     with:
       config: ${{ toJSON(matrix.config) }}
       use_chatgpt: '{"url": "${{ vars.CHATGPT_URL }}", "model": "${{ vars.CHATGPT_MODEL }}", "extra_cfg": ${{ vars.CHATGPT_EXTRA_CFG }}}'
+      kurtosis_version: ${{ needs.prepare_cache.outputs.kurtosis_version }}
     secrets:
       RANCHER_URL: ${{ secrets.RANCHER_URL }}
       RANCHER_TOKEN: ${{ secrets.RANCHER_TOKEN }}

diff --git a/.github/workflows/run-scheduled.yml b/.github/workflows/run-scheduled.yml
@@ -15,6 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       test_configs: ${{ steps.tests.outputs.test_configs }}
+      has_docker_tests: ${{ steps.tests.outputs.has_docker_tests }}
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v4
@@ -30,8 +31,20 @@ jobs:
         echo "$(echo "$test_configs" | jq)"
         echo "EOF" >> $GITHUB_OUTPUT
 
-  run_tests:
+        has_docker_tests="false"
+        if [ $(echo "$test_configs" | jq -c ".[] | select(.backend == \"docker\")" | wc -l) -gt 0 ]; then
+          has_docker_tests="true"
+        fi
+
+        echo "has_docker_tests=$has_docker_tests" >> $GITHUB_OUTPUT
+
+  prepare_cache:
     needs: get_tests
+    if: "${{ needs.get_tests.outputs.has_docker_tests == 'true' }}"
+    uses: ./.github/workflows/_shared-prepare-docker-cache.yaml
+
+  run_tests:
+    needs: [get_tests, prepare_cache]
     uses: ./.github/workflows/_shared-run.yaml
     name: "${{ matrix.config.name }}"
     strategy: