Flaky CI #57
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run all tests, including flaky test. | |
| # | |
| # The default CI workflow ignores flaky tests. This workflow will run | |
| # all tests, including ignored ones. | |
| # | |
| # To use this workflow you can either: | |
| # | |
| # - Label a PR with "flaky-test", the normal CI workflow will not run | |
| # any jobs but the jobs here will be run. Note that to merge the PR | |
| # you'll need to remove the label eventually because the normal CI | |
| # jobs are required by branch protection. | |
| # | |
| # - Manually trigger the workflow, you may choose a branch for this to | |
| # run on. | |
| # | |
| # Additionally this jobs runs once a day on a schedule. | |
| # | |
| # Currently doctests are not run by this workflow. | |
| name: Flaky CI | |
| on: | |
| pull_request: | |
| types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ] | |
| schedule: | |
| # 06:30 UTC every day | |
| - cron: '30 6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| branch: | |
| description: 'Branch to run on, defaults to main' | |
| required: true | |
| default: 'main' | |
| type: string | |
| concurrency: | |
| group: flaky-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| IROH_FORCE_STAGING_RELAYS: "1" | |
| jobs: | |
| tests: | |
| if: "contains(github.event.pull_request.labels.*.name, 'flaky-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'" | |
| uses: './.github/workflows/tests.yaml' | |
| with: | |
| flaky: true | |
| git-ref: ${{ inputs.branch }} | |
| notify: | |
| needs: tests | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Extract test results | |
| run: | | |
| printf '${{ toJSON(needs) }}\n' | |
| result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result) | |
| echo TESTS_RESULT=$result | |
| echo "TESTS_RESULT=$result" >>"$GITHUB_ENV" | |
| - name: download nextest reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-* | |
| merge-multiple: true | |
| path: nextest-results | |
| - name: create summary report | |
| id: make_summary | |
| run: | | |
| # prevent the glob expression in the loop to match on itself when the dir is empty | |
| shopt -s nullglob | |
| # to deal with multiline outputs it's recommended to use a random EOF, the syntax is based on | |
| # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | |
| EOF=aP51VriWCxNJ1JjvmO9i | |
| echo "summary<<$EOF" >> $GITHUB_OUTPUT | |
| echo "Flaky tests failure:" >> $GITHUB_OUTPUT | |
| echo " " >> $GITHUB_OUTPUT | |
| for report in nextest-results/*.json; do | |
| # remove the name prefix and extension, and split the parts | |
| name=$(echo ${report:16:-5} | tr _ ' ') | |
| echo $name | |
| echo "- **$name**" >> $GITHUB_OUTPUT | |
| # select the failed tests | |
| # the tests have this format "crate::module$test_name", the sed expressions remove the quotes and replace $ for :: | |
| failure=$(jq --slurp '.[] | select(.["type"] == "test" and .["event"] == "failed" ) | .["name"]' $report | sed -e 's/^"//g' -e 's/\$/::/' -e 's/"//') | |
| echo "$failure" | |
| echo "$failure" >> $GITHUB_OUTPUT | |
| done | |
| echo "" >> $GITHUB_OUTPUT | |
| echo "See https://github.com/n0-computer/iroh/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT | |
| echo "$EOF" >> $GITHUB_OUTPUT | |
| - name: Notify discord on failure | |
| uses: n0-computer/discord-webhook-notify@v1 | |
| if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }} | |
| with: | |
| text: "Flaky tests in **${{ github.repository }}**:" | |
| severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }} | |
| details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }} | |
| webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }} |