workflow.yml

name: Build
on: [push]

env:
  MATRIX_RUNNERS: 8  # do not saturate runner pool

jobs:

  Prepare-Workload:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      - uses: actions/checkout@v3
      - name: Prepare Zephyr Enviroment
        run: |
          # Basic git config to apply patches
          git config --global user.name 'CI'
          git config --global user.email '<>'

          source .env
          # Download dependencies
          ./scripts/prepare_sources.sh

      - name: Prepare Workload
        id: set-matrix
        run: |
          source .env
          ./scripts/get_boards_samples_pairs.py -c zephyr.yaml > boards_sample_pairs
          echo "matrix=`./scripts/generate_matrix.py`" >> $GITHUB_OUTPUT
          ls -lah

      - name: Cache Environment
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-build-${{ github.sha }}
          path: |
            zephyrproject/
            zephyr-sdk/
            micropython/
            boards_sample_pairs

  Build:
    needs: Prepare-Workload
    runs-on: ubuntu-latest

    strategy:
      matrix: ${{ fromJSON(needs.Prepare-Workload.outputs.matrix) }}

    permissions:
      contents: 'read'
      id-token: write

    steps:
      - uses: actions/checkout@v3
      - uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-build-${{ github.sha }}
          path: |
            zephyrproject/
            zephyr-sdk/
            micropython/
            boards_sample_pairs

      - name: Prepare build environment
        run: |
          source .env
          # Install Zephyr SDK and pipeline dependencies
          sudo apt -qq update
          sudo apt -qq install --no-install-recommends git cmake ninja-build gperf \
          ccache dfu-util device-tree-compiler wget \
          python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \
          make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 parallel

          ls -lah

          # Setup runtime (SDK)
          cd zephyr-sdk
          ./setup.sh -t all -h -c
          export ZEPHYR_SDK_INSTALL_DIR=$(pwd)
          cd -

          # Setup runtime (west)
          cd zephyrproject/zephyr
          pip3 -q install -r scripts/requirements.txt
          cd ..
          west config manifest.group-filter -- +ci,+optional
          west update
          west zephyr-export
          cd ..

          # Install build script dependencies
          pip3 -q install -r requirements.txt

          # Split wokrload between all runners
          split --numeric-suffixes=1 --suffix-length=3 -n l/${MATRIX_RUNNERS} boards_sample_pairs boards_sample_pairs-

          # List workload
          ls -l boards_sample_pairs-*

      - name: Build matrix
        run: |
          source .env
          echo Zephyr Matrix Runner @${{ matrix.runner }} of ${MATRIX_RUNNERS}

          # Assign jobs to a runner
          WORKLOAD=$(printf "%03d" ${{ matrix.runner }})
          COUNT=$(wc -l < boards_sample_pairs-${WORKLOAD})

          # Start building
          parallel -j +0 --keep-order --col-sep " " --halt now,fail=1 ./scripts/build.py --config=zephyr.yaml {} -j {#} -J ${COUNT} :::: boards_sample_pairs-${WORKLOAD}

      - name: Transfer build status results
        uses: actions/upload-artifact@v3
        with:
          name: status
          path: |
            build/**/*-result.json

      - id: gcp-auth
        if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
        name: GCP Authentcation
        uses: google-github-actions/auth@v1
        with:
          service_account: ${{ secrets.GCP_SA }}
          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          cleanup_credentials: true
          create_credentials_file: true
          export_environment_variables: false

      - name: Set up Cloud SDK
        if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
        uses: google-github-actions/setup-gcloud@v1
        env:
          GOOGLE_GHA_CREDS_PATH: ${{ steps.gcp-auth.outputs.credentials_file_path }}
        with:
          version: '>= 363.0.0'

      - name: Upload artifacts to GCP
        if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
        run: |
          source .env
          gsutil -m rsync -r build/ gs://zephyr-samples-builder/zephyr/$ZEPHYR_VERSION

  Update-latest-status:
    needs: Build
    runs-on: ubuntu-latest
    if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
    concurrency: update_latest_status

    permissions:
      contents: 'read'
      id-token: write

    steps:
      - uses: actions/checkout@v3
      - uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-build-${{ github.sha }}
          path: |
            zephyrproject/
            zephyr-sdk/
            micropython/
            boards_sample_pairs

      - id: gcp-auth
        name: GCP Authentcation
        uses: google-github-actions/auth@v1
        with:
          service_account: ${{ secrets.GCP_SA }}
          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          cleanup_credentials: true
          create_credentials_file: true
          export_environment_variables: false

      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v1
        env:
          GOOGLE_GHA_CREDS_PATH: ${{ steps.gcp-auth.outputs.credentials_file_path }}
        with:
          version: '>= 363.0.0'

      - name: Update the `latest` file in the bucket.
        run: |
          source .env
          CURRENT_LATEST=""
          LATEST_REMOTE_FILE="gs://zephyr-samples-builder/zephyr/latest"

          # If the `latest` file exists, read its contents.
          ! gsutil -q stat $LATEST_REMOTE_FILE \
            || CURRENT_LATEST=$(gsutil cat $LATEST_REMOTE_FILE)

          cd zephyrproject/zephyr
          git fetch origin main

          # if `latest doesn't exist or is older then ZEPHYR_VERSION that was built in this workflow.
          if [ -z "$CURRENT_LATEST" ] || ! git merge-base --is-ancestor "$ZEPHYR_VERSION" "$CURRENT_LATEST"; then
              echo $ZEPHYR_VERSION > latest
              gsutil cp latest $LATEST_REMOTE_FILE
          fi

  Print-status:
    needs: Build
    runs-on: ubuntu-latest

    permissions:
      contents: 'read'
      id-token: write

    steps:
      - uses: actions/checkout@v3
      - name: Download build status results
        uses: actions/download-artifact@v3
        with:
          name: status
          path: build/

      - name: Install python dependencies
        run: |
          pip3 -q install -r requirements.txt

      - name: Print status
        run: |
          source .env
          ./scripts/generate_summary.py -c zephyr.yaml >> $GITHUB_STEP_SUMMARY

      - id: gcp-auth
        if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
        name: GCP Authentcation
        uses: google-github-actions/auth@v1
        with:
          service_account: ${{ secrets.GCP_SA }}
          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
          cleanup_credentials: true
          create_credentials_file: true
          export_environment_variables: false

      - name: Set up Cloud SDK
        if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
        uses: google-github-actions/setup-gcloud@v1
        env:
          GOOGLE_GHA_CREDS_PATH: ${{ steps.gcp-auth.outputs.credentials_file_path }}
        with:
          version: '>= 363.0.0'

      - name: Upload artifacts to GCP
        if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}}
        run: |
          source .env
          gsutil cp build/result.json gs://zephyr-samples-builder/zephyr/$ZEPHYR_VERSION

      - name: Upload artifacts
        uses: actions/upload-artifact@v3
        with:
          name: status
          path: |
            build/result.json