bazel.yml

# Copyright 2022 The Magma Authors.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: AGW Build, Format & Test Bazel
on:
  # yamllint disable-line rule:truthy
  workflow_dispatch:
    inputs:
      publish_bazel_profile:
        description: 'Publish bazel profile data (default: false)'
        required: false
        default: false
        type: boolean
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
  push:
    branches:
      - master

env:
  BAZEL_BASE_IMAGE: "ghcr.io/magma/magma/bazel-base:latest"
  # see GH14041
  CACHE_KEY: bazel-base-image-sha-c4de1e5
  REMOTE_DOWNLOAD_OPTIMIZATION: true

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  path_filter:
    runs-on: ubuntu-latest
    outputs:
      files_changed: ${{ steps.changes.outputs.files_changed }}
    if: github.repository_owner == 'magma' || github.event_name == 'workflow_dispatch'
    steps:
      # Need to get git on push event
      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # pin@v2.11.1
        if: github.event_name == 'pull_request'
        id: changes
        with:
          filters: |
            files_changed:
              - '.github/workflows/bazel.yml'
              - 'orc8r/gateway/**'
              - 'orc8r/lib/go/**'
              - 'orc8r/protos/**'
              - 'feg/cloud/go/**'
              - 'feg/gateway/**'
              - 'lte/gateway/**'
              - 'lte/cloud/go/**'
              - 'lte/protos/**'
              - 'src/go/**'
              - '**/BUILD'
              - '**/*.BUILD'
              - '**/*.bazel'
              - '**/*.bzl'
              - '.bazelrc'

  bazel_build_and_test:
    needs: path_filter
    # Only run workflow if this is a push to the magma repository,
    # if the workflow has been triggered manually or if it is a pull_request.
    if: |
      (github.event_name == 'push' && github.repository_owner == 'magma') ||
      needs.path_filter.outputs.files_changed == 'true' ||
      github.event_name == 'workflow_dispatch'
    strategy:
      fail-fast: false
      matrix:
        include:
          - bazel-config: ""
            bazel-target: ".*_test"
          - bazel-config: "--config=asan"
            bazel-target: "cc_test"
          - bazel-config: "--config=production"
            bazel-target: "cc_test"
    name: Bazel Build & Test Job
    runs-on: ubuntu-latest
    steps:
      - name: Check Out Repo
        # This is necessary for overlays into the Docker container below.
        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3.1.0
      - name: Maximize build space
        uses: ./.github/workflows/composite/maximize-build-space
      - name: Setup Bazel Base Image
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          options: --pull always
          # Run a simple echo command to pull down the image. This makes it a bit more clear how much time is spent on building Magma and not pulling down the image.
          run: |
            echo "Pulled the bazel base image!"
      - name: Run bazel build and test
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          shell: bash
          # TODO: Remove work-around mount of Github workspace to /magma (https://github.com/addnab/docker-run-action/issues/11)
          options: -v ${{ github.workspace }}:/workspaces/magma/ -v ${{ github.workspace }}/lte/gateway/configs:/etc/magma
          run: |
            cd /workspaces/magma
            set -euo pipefail

            printf '\r%s\r' '###############################' 1>&2
            printf '\r%s\r' 'Configuring bazel remote cache.' 1>&2
            printf '\r%s\r\r' '###############################' 1>&2
            bazel/scripts/remote_cache_bazelrc_setup.sh "${{ env.CACHE_KEY }}" "${{ env.REMOTE_DOWNLOAD_OPTIMIZATION }}" "${{ secrets.BAZEL_REMOTE_PASSWORD }}"

            printf '\r%s\r' '###############################' 1>&2
            printf '\r%s\r' 'Executing bazel test ${{ matrix.bazel-config }}' 1>&2
            printf '\r%s\r' '###############################' 1>&2
            TEST_FAILED="false"
            bazel test \
              `bazel query 'kind(${{ matrix.bazel-target }}, //...) except attr("tags", "manual", //...)'` \
              ${{ matrix.bazel-config }} \
              --test_output=errors \
              --profile=Bazel_test_all_profile || TEST_FAILED="true"
            # Create Bazel unit-test results
            # Can't be a separate step, because the container's '/tmp' folder is not preserved between steps
            mkdir bazel_unit_test_results/
            UNIQUE_FILENAME_INDEX=0
            TEST_REPORT_PATHS=( $(find bazel-testlogs/ -name 'test.xml') )
            for TEST_REPORT_PATH in "${TEST_REPORT_PATHS[@]}"
            do
              cp "${TEST_REPORT_PATH}" "bazel_unit_test_results/test_result_${UNIQUE_FILENAME_INDEX}.xml"
              UNIQUE_FILENAME_INDEX=$((UNIQUE_FILENAME_INDEX + 1))
            done

            if [[ "${TEST_FAILED}" == "true" ]];
            then
              echo "ERROR: 'bazel test' failed!"
              exit 1
            fi
      - name: Create merged test-result XML file
        if: always()
        run: |
          mkdir -p lte/gateway/test_results/
          python3 lte/gateway/python/scripts/runtime_report.py -i "[^\/]+\.xml" -w "bazel_unit_test_results" -o "lte/gateway/test_results/merged_unit_test_reports.xml"
      - name: Publish bazel test profile
        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # pin@v3
        if: ${{ always() && github.event.inputs.publish_bazel_profile == 'true' }}
        with:
          name: Bazel test all profile ${{ matrix.bazel-config }}
          path: Bazel_test_all_profile
      - name: Upload Bazel unit-test results ${{ matrix.bazel-config }}
        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # pin@v3
        if: always()
        with:
          name: Bazel unit-test results ${{ matrix.bazel-config }}
          path: lte/gateway/test_results/merged_unit_test_reports.xml
      - name: Publish Bazel unit-test results ${{ matrix.bazel-config }}
        if: always()
        uses: EnricoMi/publish-unit-test-result-action/composite@46ab8d49369d898e381a607119161771bc65c2a6 # pin@v2.2.0
        with:
          check_name: Bazel unit-test results ${{ matrix.bazel-config }}
          junit_files: lte/gateway/test_results/**/*.xml
          check_run_annotations: all tests
      - name: Build space left after run
        shell: bash
        run: |
          echo "Available storage:"
          df -h
      - name: Notify failure to slack
        if: failure() && github.event_name == 'push' && github.repository_owner == 'magma'
        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # pin@v2.2.0
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: "Bazel Build & Test Job `bazel test //...` ${{ matrix.bazel-config }}"
          SLACK_USERNAME: ${{ github.workflow }}
          SLACK_ICON_EMOJI: ":boom:"
          SLACK_COLOR: "#FF0000"
          SLACK_FOOTER: ' '
          MSG_MINIMAL: actions url,commit

  if_bazel_build_and_test_success:
    name: Run when bazel successful
    needs: bazel_build_and_test
    runs-on: ubuntu-latest
    if: success() # Only run after all matrix jobs have passed
    # See https://github.com/magma/magma/wiki/How-to-set-up-a-required-matrix-workflow-on-GitHub-actions
    # or https://github.com/magma/magma/pull/13562 for more details.
    # Store success output flag for report_result_bazel_build_and_test job
    outputs:
      success: ${{ steps.setoutput.outputs.success }}
    steps:
      - id: setoutput
        run: echo "success=true" >> $GITHUB_OUTPUT

  report_result_bazel_build_and_test:
    name: Bazel build and test status
    runs-on: ubuntu-latest
    if: always()
    # This job always needs to run. It will be green if the bazel_build_and_test
    # job was successful in all matrix jobs or if the job was skipped.
    needs: [path_filter, bazel_build_and_test, if_bazel_build_and_test_success]
    steps:
      - run: |
          bazel_build_and_test_success="${{ needs.if_bazel_build_and_test_success.outputs.success }}"
          files_changed="${{ needs.path_filter.outputs.files_changed }}"

          echo "The status of this job is determined by the statuses of the previous jobs in this workflow."
          echo "For more details on this matrix workflow please look at the following wiki page or the PR #13562:"
          echo "https://github.com/magma/magma/wiki/How-to-set-up-a-required-matrix-workflow-on-GitHub-actions"
          echo "https://github.com/magma/magma/pull/13562"

          echo "bazel_build_and_test_success: $bazel_build_and_test_success"
          echo "files_changed: $files_changed"

          if [[ $bazel_build_and_test_success == "true" || $files_changed != "true" ]];
          then
            echo "Bazel build and test job passed or was skipped"
            exit 0
          else
            echo "Bazel build and test job failed"
            exit 1
          fi

  buildifier:
    needs: path_filter
    # Only run workflow if this is a push to the magma repository,
    # if the workflow has been triggered manually or if it is a pull_request.
    if: |
      (github.event_name == 'push' && github.repository_owner == 'magma') ||
      needs.path_filter.outputs.files_changed == 'true' ||
      github.event_name == 'workflow_dispatch'
    name: Bazel Starlark Format Job
    runs-on: ubuntu-latest
    steps:
      - name: Check Out Repo
        # This is necessary for overlays into the Docker container below.
        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3.1.0
      - name: Setup Bazel Base Image
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          options: --pull always
          # Run a simple echo command to pull down the image. This makes it a bit more clear how much time is spent on building Magma and not pulling down the image.
          run: |
            echo "Pulled the bazel base image!"
      - name: Run starlark format check
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          shell: bash
          # TODO: Remove work-around mount of Github workspace to /magma (https://github.com/addnab/docker-run-action/issues/11)
          options: -v ${{ github.workspace }}:/workspaces/magma/ -v ${{ github.workspace }}/lte/gateway/configs:/etc/magma
          run: |
            cd /workspaces/magma
            bazel run //:check_starlark_format
      - name: Notify failure to slack
        if: failure() && github.event_name == 'push' && github.repository_owner == 'magma'
        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # pin@v2.2.0
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: "Bazel starlark format check"
          SLACK_USERNAME: ${{ github.workflow }}
          SLACK_ICON_EMOJI: ":boom:"
          SLACK_COLOR: "#FF0000"
          SLACK_FOOTER: ' '
          MSG_MINIMAL: actions url,commit

  python_import_check:
    needs: path_filter
    # Only run workflow if this is a push to the magma repository,
    # if the workflow has been triggered manually or if it is a pull_request.
    if: |
      (github.event_name == 'push' && github.repository_owner == 'magma') ||
      needs.path_filter.outputs.files_changed == 'true' ||
      github.event_name == 'workflow_dispatch'
    name: Bazel Python Import Check
    runs-on: ubuntu-latest
    steps:
      - name: Check Out Repo
        # This is necessary for overlays into the Docker container below.
        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3.1.0
      - name: Maximize build space
        uses: ./.github/workflows/composite/maximize-build-space
      - name: Setup Bazel Base Image
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          options: --pull always
          # Run a simple echo command to pull down the image. This makes it a bit more clear how much time is spent on building Magma and not pulling down the image.
          run: |
            echo "Pulled the bazel base image!"
      - name: Run Python Import Check
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          shell: bash
          # TODO: Remove work-around mount of Github workspace to /magma (https://github.com/addnab/docker-run-action/issues/11)
          options: -v ${{ github.workspace }}:/workspaces/magma/ -v ${{ github.workspace }}/lte/gateway/configs:/etc/magma
          run: |
            cd /workspaces/magma
            set -euo pipefail

            printf '\r%s\r' '###############################' 1>&2
            printf '\r%s\r' 'Configuring bazel remote cache.' 1>&2
            printf '\r%s\r\r' '###############################' 1>&2
            bazel/scripts/remote_cache_bazelrc_setup.sh "${{ env.CACHE_KEY }}" "${{ env.REMOTE_DOWNLOAD_OPTIMIZATION }}" "${{ secrets.BAZEL_REMOTE_PASSWORD }}"

            printf '\r%s\r' '###############################' 1>&2
            printf '\r%s\r' 'Executing python import bazelification check.' 1>&2
            printf '\r%s\r' '###############################' 1>&2
            bazel/scripts/test_python_service_imports.sh;
      - name: Build space left after run
        shell: bash
        run: |
          echo "Available storage:"
          df -h
      - name: Notify failure to slack
        if: failure() && github.event_name == 'push' && github.repository_owner == 'magma'
        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # pin@v2.2.0
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: "Bazel Python Import Check"
          SLACK_USERNAME: ${{ github.workflow }}
          SLACK_ICON_EMOJI: ":boom:"
          SLACK_COLOR: "#FF0000"
          SLACK_FOOTER: ' '
          MSG_MINIMAL: actions url,commit

  bazel_package:
    needs: path_filter
    # Only run workflow if this is a push to the magma repository,
    # if the workflow has been triggered manually or if it is a pull_request.
    if: |
      (github.event_name == 'push' && github.repository_owner == 'magma') ||
      needs.path_filter.outputs.files_changed == 'true' ||
      github.event_name == 'workflow_dispatch'
    name: Bazel Package Job
    runs-on: ubuntu-latest
    steps:
      - name: Check Out Repo
        # This is necessary for overlays into the Docker container below.
        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3.1.0
      - name: Maximize build space
        uses: ./.github/workflows/composite/maximize-build-space
      - name: Setup Bazel Base Image
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          options: --pull always
          # Run a simple echo command to pull down the image. This makes it a bit more clear how much time is spent on building Magma and not pulling down the image.
          run: |
            echo "Pulled the bazel base image!"
      - name: Build .deb Packages
        uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # pin@v3
        with:
          image: ${{ env.BAZEL_BASE_IMAGE }}
          # TODO: Remove work-around mount of Github workspace to /magma (https://github.com/addnab/docker-run-action/issues/11)
          options: -v ${{ github.workspace }}:/workspaces/magma/ -v ${{ github.workspace }}/lte/gateway/configs:/etc/magma
          run: |
            cd /workspaces/magma
            bazel/scripts/remote_cache_bazelrc_setup.sh "${{ env.CACHE_KEY }}" "${{ env.REMOTE_DOWNLOAD_OPTIMIZATION }}" "${{ secrets.BAZEL_REMOTE_PASSWORD }}"
            bazel run //lte/gateway/release:release_build \
              --config=production \
              --profile=Bazel_build_package_profile
            mkdir packages
            mv /tmp/packages/*.deb packages
      - name: Get magma version
        run: |
          version_pattern="magma_([0-9]+\.[0-9]+\.[0-9]+-[0-9]+-[a-z0-9]+)_amd64.deb"
          for i in packages/*.deb; do
              if [[ $i =~ $version_pattern ]]; then
                  magma_version=${BASH_REMATCH[1]}
              fi
          done
          if [[ -z "$magma_version" ]]; then
              echo "No file found with a matching version pattern \"${version_pattern}\". Files in folder:"
              ls -la packages/*.deb
              exit 1
          else
              echo "Exporting magma version \"${magma_version}\""
              echo "MAGMA_VERSION=${magma_version}" >> $GITHUB_ENV
          fi

      - name: Setup JFROG CLI
        uses: jfrog/setup-jfrog-cli@d0a59b1cdaeeb16e65b5039fc92b8507337f1559 # pin@v3

      - name: Set dry run environment variable
        if: ${{ ! ( github.event_name == 'push' && github.repository_owner == 'magma' && github.ref_name == 'master' ) }}
        run: |
          echo "IS_DRY=--dry-run" >> $GITHUB_ENV

      - name: Publish debian packages - magmacore
        env:
          DEBIAN_META_INFO: deb.distribution=focal-ci;deb.component=main;deb.architecture=amd64
        run: |
          jf rt upload \
            --recursive=false \
            --detailed-summary \
            --url https://artifactory.magmacore.org:443/artifactory/ \
            --user ${{ secrets.JFROG_USERNAME }} \
            --password ${{ secrets.JFROG_PASSWORD }} \
            ${{ env.IS_DRY }} \
            --target-props="${DEBIAN_META_INFO}" \
            "packages/(*).deb" debian-test/pool/focal-ci/{1}.deb

      - name: Publish debian packages - linuxfoundation
        env:
          DEBIAN_META_INFO: deb.distribution=focal-ci;deb.component=main;deb.architecture=amd64
        run: |
          jf rt upload \
            --recursive=false \
            --detailed-summary \
            --url https://linuxfoundation.jfrog.io/artifactory/ \
            --user ${{ secrets.LF_JFROG_USERNAME }} \
            --password ${{ secrets.LF_JFROG_PASSWORD }} \
            ${{ env.IS_DRY }} \
            --target-props="${DEBIAN_META_INFO}" \
            "packages/(*).deb" magma-packages-test/pool/focal-ci/{1}.deb

      - name: Trigger debian integ test workflow
        uses: peter-evans/repository-dispatch@f2696244ec00ed5c659a5cc77f7138ad0302dffb # pin@v2.1.0
        if: github.event_name == 'push' && github.repository_owner == 'magma' && github.ref_name == 'master'
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          repository: magma/magma
          event-type: magma-debian-artifact
          client-payload: '{ "magma_version": "${{ env.MAGMA_VERSION }}" }'

      - name: Publish bazel profile
        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # pin@v3
        if: ${{ always() && github.event.inputs.publish_bazel_profile == 'true' }}
        with:
          name: Bazel build package profile
          path: Bazel_build_package_profile
      - name: Build space left after run
        shell: bash
        run: |
          echo "Available storage:"
          df -h
      - name: Notify failure to slack
        if: failure() && github.event_name == 'push' && github.repository_owner == 'magma'
        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # pin@v2.2.0
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: "Bazel Package Job"
          SLACK_USERNAME: ${{ github.workflow }}
          SLACK_ICON_EMOJI: ":boom:"
          SLACK_COLOR: "#FF0000"
          SLACK_FOOTER: ' '
          MSG_MINIMAL: actions url,commit

  python_file_check:
    name: Check if there are not bazelified python files
    runs-on: ubuntu-latest
    steps:
      - name: Check Out Repo
        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3.1.0
      - name: Execute check
        shell: bash
        run: |
          ./bazel/scripts/check_py_bazel.sh
      - name: Notify failure to slack
        if: failure() && github.event_name == 'push' && github.repository_owner == 'magma'
        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # pin@v2.2.0
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: "Bazel Python Check Job `./bazel/scripts/check_py_bazel.sh`"
          SLACK_USERNAME: ${{ github.workflow }}
          SLACK_ICON_EMOJI: ":boom:"
          SLACK_COLOR: "#FF0000"
          SLACK_FOOTER: ' '
          MSG_MINIMAL: actions url,commit

  c_cpp_file_check:
    name: Check if there are non-bazelified c or c++ files
    runs-on: ubuntu-latest
    steps:
      - name: Check Out Repo
        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # pin@v3.1.0
      - name: Execute check
        shell: bash
        run: |
          ./bazel/scripts/check_c_cpp_bazel.sh
      - name: Notify failure to slack
        if: failure() && github.event_name == 'push' && github.repository_owner == 'magma'
        uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 # pin@v2.2.0
        env:
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
          SLACK_TITLE: "Bazel C/C++ Check Job `./bazel/scripts/check_c_cpp_bazel.sh`"
          SLACK_USERNAME: ${{ github.workflow }}
          SLACK_ICON_EMOJI: ":boom:"
          SLACK_COLOR: "#FF0000"
          SLACK_FOOTER: ' '
          MSG_MINIMAL: actions url,commit