Stackable Build Pipeline #1135
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Stackable Build Pipeline | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - staging | |
| - trying | |
| - "renovate/**" | |
| tags: | |
| - '[0-9][0-9].[0-9]+.[0-9]+' | |
| pull_request: | |
| merge_group: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_INCREMENTAL: '0' | |
| CARGO_PROFILE_DEV_DEBUG: '0' | |
| RUSTFLAGS: "-D warnings" | |
| RUSTDOCFLAGS: "-D warnings" | |
| RUST_LOG: "info" | |
| DEV_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-dev | |
| TEST_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-test | |
| STABLE_REPO_HELM_URL: https://repo.stackable.tech/repository/helm-stable | |
| jobs: | |
| # Identify unused dependencies | |
| run_udeps: | |
| name: Run Cargo Udeps | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_BOOTSTRAP: 1 | |
| steps: | |
| - name: Install host dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config | |
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/[email protected] | |
| - uses: Swatinem/rust-cache@dd05243424bd5c0e585e4b55eb2d7615cdd32f1f # v2 | |
| with: | |
| key: udeps | |
| cache-all-crates: "true" | |
| - run: cargo install --locked [email protected] | |
| - run: cargo udeps --workspace | |
| # This job evaluates the github environment to determine why this action is running and selects the appropriate | |
| # target repository for published Helm charts based on this. | |
| # | |
| # The following scenarios are identified: | |
| # - all pull requests land in the test repository: | |
| # condition: github.event_name == "pull_request" | |
| # repository: test | |
| # | |
| # - all tagged releases land in stable: | |
| # condition: github.event_name == 'create' & github.ref.startswith('refs/tags/') | |
| # repository: stable | |
| # | |
| # - all pushes to main (i.e. PR-merges) land in dev: | |
| # condition: github.event_name == 'push' & github.ref == 'refs/heads/main' | |
| # repository: dev | |
| # | |
| # Any other scenarios (e.g. when a branch is created/pushed) will cause the publish step to be skipped, most commonly this is expected to happen for the | |
| # branches that the GitHub merge queue feature uses internally for which the checks need to run, but we do not want artifacts to be published. | |
| select_helm_repo: | |
| name: Select target helm repository based on action trigger | |
| runs-on: ubuntu-latest | |
| outputs: | |
| helm_repository: ${{ steps.selecthelmrepo.outputs.helm_repo }} | |
| steps: | |
| - id: selecthelmrepo | |
| env: | |
| TRIGGER: ${{ github.event_name }} | |
| GITHUB_REF: ${{ github.ref }} | |
| run: | | |
| if [[ $TRIGGER == "pull_request" ]]; then | |
| echo "exporting test as target helm repo: ${{ env.TEST_REPO_HELM_URL }}" | |
| echo "helm_repo=${{ env.TEST_REPO_HELM_URL }}" >> $GITHUB_OUTPUT | |
| elif [[ $TRIGGER == "push" && $GITHUB_REF == "refs/heads/main" ]]; then | |
| echo "exporting dev as target helm repo: ${{ env.DEV_REPO_HELM_URL }}" | |
| echo "helm_repo=${{ env.DEV_REPO_HELM_URL }}" >> $GITHUB_OUTPUT | |
| elif [[ ( $TRIGGER == "create" || $TRIGGER == "push" ) && $GITHUB_REF == refs/tags/* ]]; then | |
| echo "exporting stable as target helm repo: ${{ env.STABLE_REPO_HELM_URL }}" | |
| echo "helm_repo=${{ env.STABLE_REPO_HELM_URL }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "Unknown trigger and ref combination encountered, skipping publish step: $TRIGGER $GITHUB_REF" | |
| echo "helm_repo=skip" >> $GITHUB_OUTPUT | |
| fi | |
| run_cargodeny: | |
| name: Run Cargo Deny | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| checks: | |
| - advisories | |
| - bans licenses sources | |
| # Prevent sudden announcement of a new advisory from failing ci: | |
| continue-on-error: ${{ matrix.checks == 'advisories' }} | |
| steps: | |
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: EmbarkStudios/cargo-deny-action@a50c7d5f86370e02fae8472c398f15a36e517bb8 # v1 | |
| with: | |
| command: check ${{ matrix.checks }} | |
| run_rustfmt: | |
| name: Run Rustfmt | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/[email protected] | |
| with: | |
| components: rustfmt | |
| - run: cargo fmt --all -- --check | |
| run_clippy: | |
| name: Run Clippy | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config | |
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/[email protected] | |
| with: | |
| components: clippy | |
| - uses: Swatinem/rust-cache@dd05243424bd5c0e585e4b55eb2d7615cdd32f1f # v2 | |
| with: | |
| key: clippy | |
| cache-all-crates: "true" | |
| - name: Run clippy action to produce annotations | |
| uses: giraffate/clippy-action@13b9d32482f25d29ead141b79e7e04e7900281e0 # v1.0.1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: env.GITHUB_TOKEN != null | |
| with: | |
| clippy_flags: --all-targets -- -D warnings | |
| reporter: 'github-pr-review' | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run clippy manually without annotations | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: env.GITHUB_TOKEN == null | |
| run: cargo clippy --color never -q --all-targets -- -D warnings | |
| run_rustdoc: | |
| name: Run RustDoc | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config | |
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/[email protected] | |
| with: | |
| components: rustfmt | |
| - uses: Swatinem/rust-cache@dd05243424bd5c0e585e4b55eb2d7615cdd32f1f # v2 | |
| with: | |
| key: doc | |
| cache-all-crates: "true" | |
| - run: cargo doc --document-private-items | |
| run_tests: | |
| name: Run Cargo Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config | |
| - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/[email protected] | |
| - uses: Swatinem/rust-cache@dd05243424bd5c0e585e4b55eb2d7615cdd32f1f # v2 | |
| with: | |
| key: test | |
| cache-all-crates: "true" | |
| - run: cargo test | |
| # Similar to check_charts, this tries to render the README, and see if there are unintended changes. | |
| # This will save us from merging changes to the wrong file (instead of the templated source), and from | |
| # forgetting to render out modifications to the README. | |
| check_readme: | |
| name: Check if committed README is the one we would render from the available parts | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install jinja2-cli | |
| run: pip install jinja2-cli==0.8.2 | |
| - name: Regenerate charts | |
| run: make render-readme | |
| - name: Check if committed README were up to date | |
| run: git diff --exit-code | |
| - name: Git Diff showed uncommitted changes | |
| if: ${{ failure() }} | |
| uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6 | |
| with: | |
| script: | | |
| core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!') | |
| # This job cleans up the CRDs and Helm charts, followed by rebuilding them | |
| # It then runs a `git diff` and fails the entire workflow, if any difference is encountered. | |
| # | |
| # Since CRD files are generated during the 'cargo build' process we need to run this once after | |
| # removing the CRD files to ensure that the checked in versions match what the code expects. | |
| # | |
| # The reason for this step is, that developers are expected to check in up-to-date versions of charts | |
| # as we'd otherwise have to build these in CI and commit them back to the PR, which | |
| # creates all kinds of problems. | |
| # This failsafe simply aborts anything that has not had charts rebuilt before pushing. | |
| check_charts: | |
| name: Check if committed Helm charts are up to date | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install host dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config | |
| - name: Checkout | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - name: Set up Helm | |
| uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 | |
| with: | |
| version: v3.6.2 | |
| - name: Set up cargo | |
| uses: dtolnay/[email protected] | |
| - name: Set up rust-cache | |
| uses: Swatinem/rust-cache@dd05243424bd5c0e585e4b55eb2d7615cdd32f1f # v2 | |
| with: | |
| key: charts | |
| cache-all-crates: "true" | |
| - name: Regenerate charts | |
| run: make regenerate-charts | |
| - name: Check if committed charts were up to date | |
| run: git diff --exit-code | |
| - name: Git Diff showed uncommitted changes | |
| if: ${{ failure() }} | |
| uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6 | |
| with: | |
| script: | | |
| core.setFailed('Committed charts were not up to date, please regenerate and re-commit!') | |
| tests_passed: | |
| name: All tests passed | |
| needs: | |
| - run_udeps | |
| - run_cargodeny | |
| - run_clippy | |
| - run_rustfmt | |
| - run_rustdoc | |
| - run_tests | |
| - check_charts | |
| - check_readme | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: log | |
| run: echo All tests have passed! | |
| package_and_publish: | |
| name: Package Charts, Build Docker Image and publish them | |
| needs: | |
| - tests_passed | |
| - select_helm_repo | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| env: | |
| NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} | |
| HELM_REPO: ${{ needs.select_helm_repo.outputs.helm_repository }} | |
| if: needs.select_helm_repo.outputs.helm_repository != 'skip' | |
| outputs: | |
| IMAGE_TAG: ${{ steps.printtag.outputs.IMAGE_TAG }} | |
| steps: | |
| - name: Install host dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config | |
| - name: Checkout | |
| uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3 | |
| with: | |
| submodules: recursive | |
| - uses: dtolnay/[email protected] | |
| with: | |
| components: rustfmt | |
| # This step checks if the current run was triggered by a push to a pr (or a pr being created). | |
| # If this is the case it changes the version of this project in all Cargo.toml files to include the suffix | |
| # "-pr<prnumber>" so that the published artifacts can be linked to this PR. | |
| - name: Update version if PR | |
| if: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| cargo install cargo-edit --version 0.11.11 | |
| cargo set-version --offline --workspace 0.0.0-pr${{ github.event.pull_request.number }} | |
| # Recreate charts and publish charts and docker image. The "-e" is needed as we want to override the | |
| # default value in the makefile if called from this action, but not otherwise (i.e. when called locally). | |
| # This is needed for the HELM_REPO variable. | |
| - name: Set up Cosign | |
| uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 # v3 | |
| - name: Publish Docker image and Helm chart | |
| run: make -e publish | |
| # Output the name of the published image to the Job output for later use | |
| - id: printtag | |
| name: Output image name and tag | |
| run: echo "IMAGE_TAG=$(make -e print-docker-tag)" >> $GITHUB_OUTPUT | |
| openshift_preflight: | |
| name: Run the OpenShift Preflight check on the published images | |
| needs: | |
| - package_and_publish | |
| runs-on: ubuntu-latest | |
| env: | |
| IMAGE_TAG: ${{ needs.package_and_publish.outputs.IMAGE_TAG }} | |
| steps: | |
| - name: Install preflight | |
| run: | | |
| wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/latest/download/preflight-linux-amd64 | |
| chmod +x preflight-linux-amd64 | |
| - name: Check container | |
| run: ./preflight-linux-amd64 check container "$IMAGE_TAG" > preflight.out | |
| - name: "Passed?" | |
| run: '[ "$(./preflight-linux-amd64 check container "$IMAGE_TAG" | jq -r .passed)" == true ]' |