CI Main #37
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Main | |
| on: | |
| merge_group: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 * * * *" | |
| pull_request: | |
| # runs for the same workflow are cancelled on PRs but not on master | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: read-all | |
| env: | |
| CI_COMMIT_SHA: ${{ github.sha }} | |
| CI_COMMIT_REF_PROTECTED: ${{ github.ref_protected }} | |
| CI_JOB_NAME: ${{ github.job }} | |
| CI_JOB_ID: ${{ github.job }} # github does not expose this variable https://github.com/orgs/community/discussions/8945 | |
| CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| CI_PIPELINE_SOURCE: ${{ github.event_name }} | |
| CI_PROJECT_DIR: ${{ github.workspace }} | |
| CI_MERGE_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }} | |
| ROOT_PIPELINE_ID: ${{ github.run_id }} | |
| BAZEL_STARTUP_ARGS: "--output_base=/var/tmp/bazel-output/" | |
| RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" | |
| AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }} | |
| DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| CI_MERGE_REQUEST_TITLE: ${{ github.event.pull_request.title }} | |
| BUILDEVENT_APIKEY: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| BUILDEVENT_DATASET: "github-ci-dfinity" | |
| jobs: | |
| bazel-test-all: | |
| name: Bazel Test All | |
| runs-on: | |
| labels: bazel-runner-large | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' }} # needed to avoid running on public dfinity org until published | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v3 # v4 does not work with bazel-runner-large | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| fetch-depth: 256 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run Bazel Test All | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//... --deleted_packages=gitlab-ci/src/gitlab_config" | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| # check if PR title contains release and set timeout filters accordingly | |
| BAZEL_EXTRA_ARGS_RULES: ${{ contains(github.event.pull_request.title, 'release') && '--test_timeout_filters=short,moderate' || '' }} | |
| BAZEL_EXTRA_ARGS: "--keep_going --verbose_failures" | |
| # run on diff only if it is a pull request, otherwise run all targets | |
| RUN_ON_DIFF_ONLY: ${{ contains(github.event_name, 'pull_request') && 'true' || 'false'}} | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Upload bazel-targets | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: bazel-targets | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| bazel-targets | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| bazel-build-all-config-check: | |
| runs-on: | |
| labels: bazel-runner-large | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' }} # needed to avoid running on public dfinity org until published | |
| name: Bazel Build All Config Check | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v3 # v4 does not work with bazel-runner-large | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| fetch-depth: 256 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run bazel build --config=check //rs/... | |
| id: bazel-build-config-check | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=check" | |
| # run on diff only if it is a pull request, otherwise run all targets | |
| RUN_ON_DIFF_ONLY: ${{ contains(github.event_name, 'pull_request') && 'true' || 'false'}} | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| bazel-test-darwin-x86-64: | |
| name: Bazel Test Darwin x86-64 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' && github.event_name != 'pull_request' && github.event_name != 'merge_group' }} # disable until we have more macos runners | |
| runs-on: | |
| labels: macos | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| filters: | | |
| bazel-test-darwin-x86-64: | |
| - '.github/workflows/main-ci.yml' | |
| - '.bazelrc' | |
| - '.bazelversion' | |
| - '**/*.bazel' | |
| - '**/*.bzl' | |
| - '**/*.lock' | |
| - '**/*.rs' | |
| - '**/*.toml' | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run Bazel Test Darwin x86-64 | |
| id: bazel-test-darwin-x86-64 | |
| if: steps.filter.outputs.bazel-test-darwin-x86-64 != 'false' | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_CI_CONFIG: "--config=ci --config macos_ci" | |
| BAZEL_COMMAND: test | |
| BAZEL_EXTRA_ARGS: "--test_tag_filters=test_macos" | |
| BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output//${ROOT_PIPELINE_ID}" | |
| BAZEL_TARGETS: "//rs/... //publish/binaries/..." | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: No run | |
| if: steps.filter.outputs.bazel-test-darwin-x86-64 == 'false' | |
| run: echo "No changes, skipping run" | |
| - name: Purge Bazel Output | |
| shell: bash | |
| run: | | |
| sudo rm -rf /var/tmp/bazel-output//${ROOT_PIPELINE_ID} | |
| bazel-build-fuzzers: | |
| name: Bazel Build Fuzzers | |
| runs-on: | |
| labels: bazel-runner-small | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Bazel Build Fuzzers | |
| id: bazel-build-fuzzers | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=ci" | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=fuzzing --build_tag_filters=libfuzzer" | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| bazel-build-fuzzers-afl: | |
| name: Bazel Build Fuzzers AFL | |
| runs-on: | |
| labels: bazel-runner-small | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Bazel Build Fuzzers AFL | |
| id: bazel-build-fuzzers-afl | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=ci" | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=afl" | |
| - name: Bazel Clean | |
| if: always() | |
| run: bazel clean | |
| python-ci-tests: | |
| name: Python CI Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run Python CI Tests | |
| id: python-ci-tests | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| export PYTHONPATH=$PWD/gitlab-ci/src:$PWD/gitlab-ci/src/dependencies | |
| pip3 install --ignore-installed -r requirements.txt | |
| cd gitlab-ci/src | |
| pytest --ignore=gitlab_config/ --ignore=git_changes/ -v -o junit_family=xunit1 \ | |
| --junitxml=../../test_report.xml --cov=. --cov-report=term \ | |
| --cov-report=term-missing --cov-report=html --cov-branch | |
| build-ic: | |
| name: Build IC | |
| runs-on: | |
| labels: bazel-runner-large | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 90 | |
| if: ${{ vars.RUN_CI == 'true' }} # needed to avoid running on public dfinity org until published | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v3 # v4 does not work with bazel-runner-large | |
| if: ${{ github.event_name == 'pull_request' }} | |
| with: | |
| fetch-depth: 256 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| if: ${{ github.event_name != 'pull_request' }} | |
| - name: Docker RO SA login | |
| id: docker-ro-login | |
| shell: bash | |
| run: | | |
| docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| if which docker-bin 2>/dev/null; then | |
| docker-bin login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD_RO" | |
| fi | |
| - name: Run Build IC | |
| id: build-ic | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| mkdir -p "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" | |
| ln -s "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" /__w/ic/ic/artifacts | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/build-ic.sh | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| env: | |
| RUN_ON_DIFF_ONLY: "true" | |
| BAZEL_COMMAND: "build" | |
| - name: Upload build-ic.tar | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-ic | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| build-ic.tar | |
| build-determinism: | |
| name: Build Determinism | |
| runs-on: | |
| labels: default-runner | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| needs: [build-ic, bazel-test-all] | |
| strategy: | |
| matrix: | |
| include: | |
| - TARGET: "//publish/binaries:upload" | |
| PATH0: "release" | |
| PATH1: "build-ic/release" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//publish/canisters:upload" | |
| PATH0: "canisters" | |
| PATH1: "build-ic/canisters" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/guestos/envs/prod:upload_disk-img" | |
| PATH0: "guest-os/update-img" | |
| PATH1: "build-ic/icos/guestos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/hostos/envs/prod:upload_update-img" | |
| PATH0: "host-os/update-img" | |
| PATH1: "build-ic/icos/hostos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/setupos/envs/prod:upload_disk-img" | |
| PATH0: "setup-os/disk-img" | |
| PATH1: "build-ic/icos/setupos" | |
| SETUPOS_FLAG: "true" | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download bazel-targets [bazel-test-all] | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: bazel-targets | |
| - name: Download build-ic.tar [build-ic] | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: build-ic | |
| - name: Build Determinism Test | |
| id: build-determinism | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| git config --global safe.directory '*' | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/build-determinism.sh | |
| env: | |
| TARGET: ${{ matrix.TARGET }} | |
| PATH0: ${{ matrix.PATH0 }} | |
| PATH1: ${{ matrix.PATH1 }} | |
| SETUPOS_FLAG: ${{ matrix.SETUPOS_FLAG }} | |
| cargo-clippy-linux: | |
| name: Cargo Clippy Linux | |
| runs-on: | |
| labels: bazel-runner-small | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| timeout-minutes: 30 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Clippy Linux | |
| id: cargo-clippy-linux | |
| if: steps.filter.outputs.cargo == 'true' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/gitlab-ci/src/ci-scripts/rust-lint.sh | |
| - name: No run | |
| if: steps.filter.outputs.cargo == 'false' | |
| run: echo "No cargo changes, skipping run" | |
| cargo-build-release-linux: | |
| name: Cargo Build Release Linux | |
| runs-on: | |
| labels: bazel-runner-small | |
| container: | |
| image: ghcr.io/dfinity-sandbox/ic-build@sha256:994f76fcc90400ec9f8be3d540bd529ff1b3768b7f7f6963887022c57ec1ab24 | |
| if: ${{ vars.RUN_CI == 'true' }} | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Before script | |
| if: always() | |
| id: before-script | |
| shell: bash | |
| run: | | |
| if [ -n "${NODE_NAME:-}" ]; then echo "Node: $NODE_NAME"; fi | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Build Release Linux | |
| id: cargo-build-release-linux | |
| if: steps.filter.outputs.cargo == 'true' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| cargo build --release | |
| - name: No run | |
| if: steps.filter.outputs.cargo == 'false' | |
| run: echo "No cargo changes, skipping run" |