split the work into more jobs #167
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: [push] | |
| env: | |
| MATRIX_RUNNERS: 10 # do not saturate runner pool | |
| jobs: | |
| Prepare-Workload: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Prepare Zephyr Enviroment | |
| run: | | |
| # Basic git config to apply patches | |
| git config --global user.name 'CI' | |
| git config --global user.email '<>' | |
| source .env | |
| # Download dependencies | |
| ./scripts/prepare_sources.sh | |
| - name: Prepare Workload | |
| id: set-matrix | |
| run: | | |
| source .env | |
| ./scripts/get_boards_samples_pairs.py -c zephyr.yaml > boards_sample_pairs | |
| echo "matrix=`./scripts/generate_matrix.py`" >> $GITHUB_OUTPUT | |
| ls -lah | |
| - name: Cache Environment | |
| uses: actions/cache@v3 | |
| with: | |
| key: ${{ runner.os }}-build-${{ github.sha }} | |
| path: | | |
| zephyrproject/ | |
| zephyr-sdk/ | |
| micropython/ | |
| boards_sample_pairs | |
| Build: | |
| needs: Prepare-Workload | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{ fromJSON(needs.Prepare-Workload.outputs.matrix) }} | |
| permissions: | |
| contents: 'read' | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/cache@v3 | |
| with: | |
| key: ${{ runner.os }}-build-${{ github.sha }} | |
| path: | | |
| zephyrproject/ | |
| zephyr-sdk/ | |
| micropython/ | |
| boards_sample_pairs | |
| - name: Prepare build environment | |
| run: | | |
| source .env | |
| # Install Zephyr SDK and pipeline dependencies | |
| sudo apt -qq update | |
| sudo apt -qq install --no-install-recommends git cmake ninja-build gperf \ | |
| ccache dfu-util device-tree-compiler wget \ | |
| python3-dev python3-pip python3-setuptools python3-tk python3-wheel xz-utils file \ | |
| make gcc gcc-multilib g++-multilib libsdl2-dev libmagic1 parallel | |
| ls -lah | |
| # Setup runtime (SDK) | |
| cd zephyr-sdk | |
| ./setup.sh -t all -h -c | |
| export ZEPHYR_SDK_INSTALL_DIR=$(pwd) | |
| cd - | |
| # Setup runtime (west) | |
| cd zephyrproject/zephyr | |
| pip3 -q install -r scripts/requirements.txt | |
| cd .. | |
| west zephyr-export | |
| cd .. | |
| # Install build script dependencies | |
| pip3 -q install -r requirements.txt | |
| # Split wokrload between all runners | |
| split --numeric-suffixes=1 --suffix-length=3 -n l/${MATRIX_RUNNERS} boards_sample_pairs boards_sample_pairs- | |
| # List workload | |
| ls -l boards_sample_pairs-* | |
| - name: Build matrix | |
| run: | | |
| source .env | |
| echo Zephyr Matrix Runner @${{ matrix.runner }} of ${MATRIX_RUNNERS} | |
| # Assign jobs to a runner | |
| WORKLOAD=$(printf "%03d" ${{ matrix.runner }}) | |
| COUNT=$(wc -l < boards_sample_pairs-${WORKLOAD}) | |
| # Start building | |
| parallel -j +0 --keep-order --col-sep " " --halt now,fail=1 ./scripts/build.py --config=zephyr.yaml {} -j {#} -J ${COUNT} :::: boards_sample_pairs-${WORKLOAD} | |
| - name: Transfer build status results | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: status | |
| path: | | |
| build/**/*-result.json | |
| - id: gcp-auth | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| name: GCP Authentcation | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| service_account: ${{ secrets.GCP_SA }} | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| cleanup_credentials: true | |
| create_credentials_file: true | |
| export_environment_variables: false | |
| - name: Set up Cloud SDK | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| uses: google-github-actions/setup-gcloud@v1 | |
| env: | |
| GOOGLE_GHA_CREDS_PATH: ${{ steps.gcp-auth.outputs.credentials_file_path }} | |
| with: | |
| version: '>= 363.0.0' | |
| - name: Upload artifacts to GCP | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| run: | | |
| source .env | |
| gsutil -m rsync -r build/ gs://zephyr-samples-builder/zephyr/$ZEPHYR_VERSION | |
| Update-latest-status: | |
| needs: Build | |
| runs-on: ubuntu-latest | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| concurrency: update_latest_status | |
| permissions: | |
| contents: 'read' | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/cache@v3 | |
| with: | |
| key: ${{ runner.os }}-build-${{ github.sha }} | |
| path: | | |
| zephyrproject/ | |
| zephyr-sdk/ | |
| micropython/ | |
| boards_sample_pairs | |
| - id: gcp-auth | |
| name: GCP Authentcation | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| service_account: ${{ secrets.GCP_SA }} | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| cleanup_credentials: true | |
| create_credentials_file: true | |
| export_environment_variables: false | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| env: | |
| GOOGLE_GHA_CREDS_PATH: ${{ steps.gcp-auth.outputs.credentials_file_path }} | |
| with: | |
| version: '>= 363.0.0' | |
| - name: Update the `latest` file in the bucket. | |
| run: | | |
| source .env | |
| CURRENT_LATEST="" | |
| LATEST_REMOTE_FILE="gs://zephyr-samples-builder/zephyr/latest" | |
| # If the `latest` file exists, read its contents. | |
| ! gsutil -q stat $LATEST_REMOTE_FILE \ | |
| || CURRENT_LATEST=$(gsutil cat $LATEST_REMOTE_FILE) | |
| cd zephyrproject/zephyr | |
| git fetch origin main | |
| # if `latest doesn't exist or is older then ZEPHYR_VERSION that was built in this workflow. | |
| if [ -z "$CURRENT_LATEST" ] || ! git merge-base --is-ancestor "$ZEPHYR_VERSION" "$CURRENT_LATEST"; then | |
| echo $ZEPHYR_VERSION > latest | |
| gsutil cp latest $LATEST_REMOTE_FILE | |
| fi | |
| Print-status: | |
| needs: Build | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download build status results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: status | |
| path: build/ | |
| - name: Install python dependencies | |
| run: | | |
| pip3 -q install -r requirements.txt | |
| - name: Print status | |
| run: | | |
| source .env | |
| ./scripts/generate_summary.py -c zephyr.yaml >> $GITHUB_STEP_SUMMARY | |
| - id: gcp-auth | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| name: GCP Authentcation | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| service_account: ${{ secrets.GCP_SA }} | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| cleanup_credentials: true | |
| create_credentials_file: true | |
| export_environment_variables: false | |
| - name: Set up Cloud SDK | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| uses: google-github-actions/setup-gcloud@v1 | |
| env: | |
| GOOGLE_GHA_CREDS_PATH: ${{ steps.gcp-auth.outputs.credentials_file_path }} | |
| with: | |
| version: '>= 363.0.0' | |
| - name: Upload artifacts to GCP | |
| if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} | |
| run: | | |
| source .env | |
| gsutil cp build/result.json gs://zephyr-samples-builder/zephyr/$ZEPHYR_VERSION | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: status | |
| path: | | |
| build/result.json |