Skip to content

Flaky Test Monitor #5435

Flaky Test Monitor

Flaky Test Monitor #5435

# This workflow runs all skipped (flaky) and non-skipped (regular) tests and generates a summary that's uploaded to BigQuery.
name: Flaky Test Monitor
on:
schedule:
- cron: '0 */2 * * *' # every 2 hours
push:
paths:
- '.github/workflows/flaky-test-monitor.yml'
env:
BIGQUERY_DATASET: production_src_flow_test_metrics
BIGQUERY_TABLE: skipped_tests
BIGQUERY_TABLE2: test_results
GO_VERSION: "1.20"
SKIPPED_TESTS_FILE: skipped-tests
RESULTS_FILE: test-results
COMMIT_SHA: ${{ github.sha }}
RUN_ID: ${{ github.run_id }}
JSON_OUTPUT: true
VERBOSE: true
TEST_FLAKY: true
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
create-dynamic-test-matrix:
name: Create Dynamic Test Matrix
runs-on: ubuntu-latest
outputs:
dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }}
steps:
- name: Checkout repo
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Set Test Matrix
id: set-test-matrix
run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils
unit-test:
name: Unit Tests (${{ matrix.targets.name }})
needs: create-dynamic-test-matrix
strategy:
fail-fast: false
matrix:
targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
# need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Setup tests (${{ matrix.targets.name }})
run: make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
- name: Run tests (${{ matrix.targets.name }})
run: make -es GO_TEST_PACKAGES="${{ matrix.targets.packages }}" unittest-main > test-output
timeout-minutes: 100
# test run should continue even if there are failed tests
continue-on-error: true
- name: Process test results
env:
TEST_CATEGORY: unit
uses: ./.github/workflows/actions/test-monitor-process-results
with:
gcp_sa_key: ${{ secrets.GCP_SA_KEY }}
unit-test-modules:
name: Unit Tests (Modules)
strategy:
fail-fast: false
matrix:
include:
- name: crypto
make1: -C crypto setup
make2: unittest
race: 1
test_category: unit-crypto
- name: insecure
make1: install-tools
make2: test
race: 0
test_category: unit-insecure
- name: integration
make1: install-tools
make2: test
race: 0
test_category: unit-integration
runs-on: ubuntu-latest
steps:
- name: Checkout repo
uses: actions/checkout@v3
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Setup tests (${{ matrix.name }})
run: make ${{ matrix.make1 }}
- name: Run tests (${{ matrix.name }})
env:
RACE_DETECTOR: ${{ matrix.race }}
run: make -es -C ${{ matrix.name }} ${{ matrix.make2 }} > test-output
timeout-minutes: 100
continue-on-error: true
- name: Process test results (${{ matrix.name }})
env:
TEST_CATEGORY: ${{ matrix.test_category }}
uses: ./.github/workflows/actions/test-monitor-process-results
with:
gcp_sa_key: ${{ secrets.GCP_SA_KEY }}
integration-test:
name: Integration Tests
strategy:
fail-fast: false
matrix:
include:
- target: access-tests
test_category: integration-access
- target: bft-protocol-tests
test_category: integration-bft-protocol
- target: bft-framework-tests
test_category: integration-bft-framework
- target: bft-gossipsub-tests
test_category: integration-bft-gossipsub
- target: collection-tests
test_category: integration-collection
- target: consensus-tests
test_category: integration-consensus
- target: epochs-cohort1-tests
test_category: integration-epochs
- target: epochs-cohort2-tests
test_category: integration-epochs
- target: execution-tests
test_category: integration-execution
- target: ghost-tests
test_category: integration-ghost
- target: mvp-tests
test_category: integration-mvp
- target: network-tests
test_category: integration-network
- target: verification-tests
test_category: integration-verification
- target: upgrades-tests
test_category: integration-upgrades
runs-on: ubuntu-latest
steps:
- name: Checkout repo
uses: actions/checkout@v3
with:
# all tags are needed for integration tests
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: ${{ env.GO_VERSION }}
cache: true
- name: Build relic
run: make crypto_setup_gopath
- name: Docker build
run: make docker-build-flow docker-build-flow-corrupt
- name: Run tests
run: make -es -C integration ${{ matrix.target }} > test-output
timeout-minutes: 100
continue-on-error: true
- name: Process test results
env:
TEST_CATEGORY: ${{ matrix.test_category }}
uses: ./.github/workflows/actions/test-monitor-process-results
with:
gcp_sa_key: ${{ secrets.GCP_SA_KEY }}