Consider slots which will be garbage collected soon as non-existing (… #348
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Full Run | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - grok/*/* | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*" | |
| jobs: | |
| # Tests command-line tool using tests scaffolding. | |
| ci-storage-tool-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| exec 2>&1; set -ex | |
| tests/all.sh | |
| # Tests action itself. | |
| ci-storage-action-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Disable default ci-storage remote host (store locally) | |
| run: | | |
| echo -n "" > ~/ci-storage-host | |
| - name: Create dummy files | |
| run: | | |
| echo "dummy" > dummy.txt | |
| mkdir -p dir/subdir | |
| echo "layer" > dir/subdir/layer.txt | |
| mkdir -p /tmp/dir | |
| echo "local-dir" > /tmp/dir/local-dir.txt | |
| - name: Test store | |
| uses: ./ | |
| with: | |
| action: store | |
| storage-dir: ~/storage-dir | |
| sudo: false | |
| run-before: | | |
| set -ex | |
| echo "run-before" > run-before.txt | |
| - name: Test store (layer) | |
| uses: ./ | |
| with: | |
| action: store | |
| storage-dir: ~/storage-dir | |
| layer-name: my-layer | |
| layer-include: layer.txt | |
| - name: Test store (custom local-dir) | |
| uses: ./ | |
| with: | |
| action: store | |
| storage-dir: ~/storage-dir | |
| local-dir: /tmp/dir | |
| - name: Remove dummy.txt and run-before.txt | |
| run: rm dummy.txt run-before.txt | |
| - name: Test load | |
| uses: ./ | |
| with: | |
| action: load | |
| storage-dir: ~/storage-dir | |
| - name: Check that dummy.txt and run-before.txt were restored | |
| run: | | |
| set -e | |
| ls -la ~/storage-dir/${{ github.repository }}/* | |
| [[ "$(cat dummy.txt)" == "dummy" ]] || { echo "dummy.txt must be restored"; exit 1; } | |
| [[ "$(cat run-before.txt)" == "run-before" ]] || { echo "run-before.txt must be restored"; exit 1; } | |
| - name: Remove dir/subdir/layer.txt | |
| run: rm -rf dir | |
| - name: Test load (layer) | |
| uses: ./ | |
| with: | |
| action: load | |
| storage-dir: ~/storage-dir | |
| layer-name: my-layer | |
| - name: Check that dir/subdir/layer.txt was restored, and dummy.txt still exists | |
| run: | | |
| set -e | |
| ls -la ~/storage-dir/${{ github.repository }}/* | |
| [[ "$(cat dummy.txt)" == "dummy" ]] || { echo "dummy.txt must be kept"; exit 1; } | |
| [[ "$(cat dir/subdir/layer.txt)" == "layer" ]] || { echo "layer.txt must be restored"; exit 1; } | |
| - name: Remove /tmp/dir/local-dir.txt | |
| run: rm -rf /tmp/dir/* | |
| - name: Test load (custom local-dir) | |
| uses: ./ | |
| with: | |
| action: load | |
| storage-dir: ~/storage-dir | |
| local-dir: /tmp/dir | |
| - name: Check that /tmp/dir/local-dir.txt was restored | |
| run: | | |
| set -e | |
| ls -la ~/storage-dir/${{ github.repository }}/* | |
| [[ "$(cat /tmp/dir/local-dir.txt)" == "local-dir" ]] || { echo "/tmp/dir/local-dir.txt must be restored"; exit 1; } | |
| # Tests ci-scaler logic. | |
| ci-scaler-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| exec 2>&1; set -ex | |
| docker/ci-scaler/guest/scaler/tests/all.sh | |
| env: | |
| GH_TOKEN: ${{ secrets.CI_PAT }} | |
| # Builds and boots a ci-runner container inside GitHub's infra. Once it's | |
| # settled, there is a running container with one self-hosted runner waiting | |
| # for jobs with ci-storage-test tag to pick up (based on Dockerfile image). | |
| build-and-boot-containers: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Start test Docker containers | |
| run: | | |
| exec 2>&1; set -ex | |
| cd docker | |
| # Build all containers. | |
| docker compose build --parallel | |
| # Boot ci-storage container in background. | |
| docker compose up ci-storage -d | |
| # Boot ci-runner container. It connects to ci-storage container and | |
| # load a test (non-existent) ci-storage slot from there, then register | |
| # a GitHub self-hosted runner and remain waiting for jobs. | |
| docker compose up ci-runner | |
| env: | |
| GH_TOKEN: ${{ secrets.CI_PAT }} | |
| GH_REPOSITORY: ${{ github.repository }} | |
| GH_LABELS: ${{ format('ci-storage-test-{0}-{1}', github.run_id, github.run_attempt) }} | |
| TZ: America/Los_Angeles | |
| FORWARD_HOST: host.docker.internal | |
| # Test the job with ci-storage-test tag which is initially queued, but then is | |
| # picked up by the ci-runner container booted in the previous job. In the end, | |
| # the test job sends SIGINT to the container entrypoint.sh PID, so the | |
| # container (based on Dockerfile image) shuts down gracefully. | |
| spawn-job-test: | |
| runs-on: | |
| - self-hosted | |
| - ${{ format('ci-storage-test-{0}-{1}', github.run_id, github.run_attempt) }} | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run test job inside the self-hosted runner | |
| run: echo "Hello, world!" | |
| - name: Test store using GitHub Action | |
| uses: ./ | |
| with: | |
| action: "store" | |
| - name: Kill ci-runner container | |
| run: kill -SIGINT $(cat ~guest/.entrypoint.pid) | |
| # Publishes ci-scaler image. | |
| push-ci-scaler: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| - build-and-boot-containers | |
| - spawn-job-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| dimikot/ci-scaler | |
| ghcr.io/${{ github.repository_owner }}/ci-scaler | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v5 | |
| with: | |
| context: docker/ci-scaler | |
| platforms: linux/amd64,linux/arm64,linux/arm64/v8 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| readme-filepath: docker/ci-scaler/README.md | |
| repository: dimikot/ci-scaler | |
| # Publishes ci-storage image. | |
| push-ci-storage: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| - build-and-boot-containers | |
| - spawn-job-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| dimikot/ci-storage | |
| ghcr.io/${{ github.repository_owner }}/ci-storage | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v5 | |
| with: | |
| context: docker/ci-storage | |
| platforms: linux/amd64,linux/arm64,linux/arm64/v8 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| readme-filepath: docker/ci-storage/README.md | |
| repository: dimikot/ci-storage | |
| # Publishes ci-runner image. | |
| push-ci-runner: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| - build-and-boot-containers | |
| - spawn-job-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| dimikot/ci-runner | |
| ghcr.io/${{ github.repository_owner }}/ci-runner | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v5 | |
| with: | |
| context: docker/ci-runner | |
| platforms: linux/amd64,linux/arm64,linux/arm64/v8 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| readme-filepath: docker/ci-runner/README.md | |
| repository: dimikot/ci-runner |