flaky-test-monitor.yml

# This workflow runs all skipped (flaky) and non-skipped (regular) tests and generates a summary that's uploaded to BigQuery.

name: Flaky Test Monitor

on:
  schedule:
    - cron: '0 */2 * * *' # every 2 hours
  push:
    paths:
      - '.github/workflows/flaky-test-monitor.yml'

env:
  BIGQUERY_DATASET: production_src_flow_test_metrics
  BIGQUERY_TABLE: skipped_tests
  BIGQUERY_TABLE2: test_results
  GO_VERSION: "1.20"
  SKIPPED_TESTS_FILE: skipped-tests
  RESULTS_FILE: test-results
  COMMIT_SHA: ${{ github.sha }}
  RUN_ID: ${{ github.run_id }}
  JSON_OUTPUT: true
  VERBOSE: true
  TEST_FLAKY: true

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  create-dynamic-test-matrix:
    name: Create Dynamic Test Matrix
    runs-on: ubuntu-latest
    outputs:
      dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }}
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Setup Go
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
      - name: Set Test Matrix
        id: set-test-matrix
        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils

  unit-test:
    name: Unit Tests (${{ matrix.targets.name }})
    needs: create-dynamic-test-matrix
    strategy:
      fail-fast: false
      matrix:
        targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
    # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Setup Go
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
      - name: Setup tests (${{ matrix.targets.name }})
        run: make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
      - name: Run tests (${{ matrix.targets.name }})
        run: make -es GO_TEST_PACKAGES="${{ matrix.targets.packages }}" unittest-main > test-output
        timeout-minutes: 100
        # test run should continue even if there are failed tests
        continue-on-error: true
      - name: Process test results
        env:
          TEST_CATEGORY: unit
        uses: ./.github/workflows/actions/test-monitor-process-results
        with:
          gcp_sa_key: ${{ secrets.GCP_SA_KEY }}

  unit-test-modules:
    name: Unit Tests (Modules)
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: crypto
            make1: -C crypto setup
            make2: unittest
            race: 1
            test_category: unit-crypto
          - name: insecure
            make1: install-tools
            make2: test
            race: 0
            test_category: unit-insecure
          - name: integration
            make1: install-tools
            make2: test
            race: 0
            test_category: unit-integration
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Setup Go
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
      - name: Setup tests (${{ matrix.name }})
        run: make ${{ matrix.make1 }}
      - name: Run tests (${{ matrix.name }})
        env:
          RACE_DETECTOR: ${{ matrix.race }}
        run: make -es -C ${{ matrix.name }} ${{ matrix.make2 }} > test-output
        timeout-minutes: 100
        continue-on-error: true
      - name: Process test results (${{ matrix.name }})
        env:
          TEST_CATEGORY: ${{ matrix.test_category }}
        uses: ./.github/workflows/actions/test-monitor-process-results
        with:
          gcp_sa_key: ${{ secrets.GCP_SA_KEY }}

  integration-test:
    name: Integration Tests
    strategy:
      fail-fast: false
      matrix:
        include:
          - target: access-tests
            test_category: integration-access
          - target: bft-protocol-tests
            test_category: integration-bft-protocol
          - target: bft-framework-tests
            test_category: integration-bft-framework
          - target: bft-gossipsub-tests
            test_category: integration-bft-gossipsub
          - target: collection-tests
            test_category: integration-collection
          - target: consensus-tests
            test_category: integration-consensus
          - target: epochs-cohort1-tests
            test_category: integration-epochs
          - target: epochs-cohort2-tests
            test_category: integration-epochs
          - target: execution-tests
            test_category: integration-execution
          - target: ghost-tests
            test_category: integration-ghost
          - target: mvp-tests
            test_category: integration-mvp
          - target: network-tests
            test_category: integration-network
          - target: verification-tests
            test_category: integration-verification
          - target: upgrades-tests
            test_category: integration-upgrades

    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
        with:
          # all tags are needed for integration tests
          fetch-depth: 0
      - name: Setup Go
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: true
      - name: Build relic
        run: make crypto_setup_gopath
      - name: Docker build
        run: make docker-build-flow docker-build-flow-corrupt
      - name: Run tests
        run: make -es -C integration ${{ matrix.target }} > test-output
        timeout-minutes: 100
        continue-on-error: true
      - name: Process test results
        env:
          TEST_CATEGORY: ${{ matrix.test_category }}
        uses: ./.github/workflows/actions/test-monitor-process-results
        with:
          gcp_sa_key: ${{ secrets.GCP_SA_KEY }}