Test solidity-verifier compatibility
#79
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # TODO: Reusify and combine with the `check-lurk-compiles` action and/or make a reusable open-issue action | |
| # | |
| # This workflow runs autogenerated `solidity-verifier` compatibility tests on Arecibo PRs and notifies if compatibility is broken | |
| # The workflow is intended to be a required status check only to make sure the Rust test & basic job steps work | |
| # It is NOT intended to block breaking changes from merging, only to noisily surface them for review | |
| # | |
| # If the Rust template fails to generate the Solidity test or the job errors for another reason, the workflow fails immediately | |
| # If the Solidity test fails on `pull_request`, it writes a PR comment to ensure the author/reviewer are notified | |
| # If the Solidity test fails on `merge_group`, it opens an issue in `solidity-verifier` downstream that compatibility has been broken | |
| # `merge_group` failures should only happen intentionally when breaking changes need to be merged in Arecibo | |
| # | |
| # Implementation note: | |
| # `falnyr/replace-env-vars-action`, `micalevisk/last-issue-action` and `peter-evans/create-issue-from-file` replace | |
| # equivalent functionality in `JasonEtco/create-an-issue`. We can't use the latter because it doesn't allow creating | |
| # the issue in another repo. See https://github.com/JasonEtco/create-an-issue/issues/40 | |
| name: Test `solidity-verifier` compatibility | |
| on: | |
| merge_group: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: | |
| - dev | |
| - 'feat/**' | |
| - release-candidate | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| solidity-compat: | |
| runs-on: buildjet-16vcpu-ubuntu-2204 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/ci-workflows | |
| - uses: ./.github/actions/ci-env | |
| - uses: actions/checkout@v4 | |
| - uses: dtolnay/rust-toolchain@stable | |
| - uses: taiki-e/install-action@nextest | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run Solidity test generator | |
| run: | | |
| # Runs all tests with the `test_solidity_compatibility` prefix, e.g. `test_solidity_compatibility_ipa` | |
| cargo nextest run -E 'test(test_solidity_compatibility)' --release --run-ignored all --nocapture > test-output | |
| working-directory: ${{ github.workspace }} | |
| - name: Check out `solidity-verifier` for tests | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: argumentcomputer/solidity-verifier | |
| path: solidity-verifier | |
| submodules: recursive | |
| - name: Install Foundry | |
| uses: foundry-rs/foundry-toolchain@v1 | |
| with: | |
| version: nightly | |
| - name: Prep Solidity test files | |
| run: | | |
| # Get test names from output, extracting the final word after the final `_` in the test output | |
| # E.g. `test provider::tests::ipa_pc::test::test_solidity_compatibility_ipa ... ok` returns `ipa` | |
| # Expects all tests to live in the `provider` module, but can be changed/strengthened in future | |
| TEST_NAMES=$(grep 'test provider::*' test-output | awk -F'[_(.*?)\b...]' '{ print $(NF-3) }') | |
| echo "$TEST_NAMES" | |
| # Print output of each test to `<test_name>.t.sol` file | |
| awk -v names="$TEST_NAMES" 'BEGIN { | |
| file_counter = 0 | |
| buffer = "" | |
| # Convert test names to array | |
| split(names, elements, " ") | |
| for (i in elements) { | |
| print "Element:", elements[i] | |
| } | |
| } | |
| /running 1 test/ { | |
| between = 1 | |
| buffer = "" | |
| } | |
| between { | |
| buffer = buffer $0 ORS | |
| } | |
| /^test provider.*$/ { | |
| between = 0 | |
| if (buffer != "") { | |
| ++file_counter | |
| print buffer > elements[file_counter]".t.sol" | |
| buffer = "" | |
| } | |
| }' test-output | |
| # Clean up | |
| shopt -s nullglob | |
| for file in *.t.sol; do | |
| cat $file | sed '1d' | head -n -2 > tmp.file && mv tmp.file solidity-verifier/test/$file | |
| done | |
| shopt -u nullglob | |
| working-directory: ${{ github.workspace }} | |
| - name: Run Forge tests | |
| id: solidity-test | |
| run: forge test -vvv | |
| working-directory: ${{ github.workspace }}/solidity-verifier | |
| continue-on-error: true | |
| # Prepares env vars for use in a PR comment or issue in `solidity-verifier` | |
| - name: Set env vars | |
| if: steps.solidity-test.outcome != 'success' | |
| run: | | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| COMMIT=$(echo ${{ github.event.pull_request.head.sha }} | cut -c -7) | |
| PR_NUMBER=${{ github.event.pull_request.number }} | |
| else | |
| COMMIT=$(echo ${{ github.event.merge_group.head_sha }} | cut -c -7) | |
| PR_NUMBER=$(echo ${{ github.event.merge_group.head_ref }} | sed -e 's/.*pr-\(.*\)-.*/\1/') | |
| fi | |
| GITHUB_URL=https://github.com/${{ github.repository }} | |
| WORKFLOW_URL=$GITHUB_URL/actions/runs/${{ github.run_id }} | |
| echo "WORKFLOW_FILE=$WORKFLOW_URL/workflow" | tee -a $GITHUB_ENV | |
| echo "WORKFLOW_URL=$WORKFLOW_URL" | tee -a $GITHUB_ENV | |
| echo "COMMIT_URL=$GITHUB_URL/commit/$COMMIT" | tee -a $GITHUB_ENV | |
| echo "PR_URL=$GITHUB_URL/pull/$PR_NUMBER" | tee -a $GITHUB_ENV | |
| echo "COMMIT=$COMMIT" | tee -a $GITHUB_ENV | |
| # Comment on PR when test fails on `pull_request` | |
| - name: Comment on failing run | |
| if: steps.solidity-test.outcome != 'success' && github.event_name == 'pull_request' | |
| uses: peter-evans/create-or-update-comment@v4 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number }} | |
| body: | | |
| `solidity-verifier` compatibility test failed :x: | |
| ${{ env.WORKFLOW_URL }} | |
| # Substitutes env vars for their values in `SOLIDITY_COMPAT_ISSUE.md` | |
| - uses: falnyr/replace-env-vars-action@master | |
| if: steps.solidity-test.outcome != 'success' && (github.event_name != 'pull_request' || github.event.action == 'enqueued') | |
| env: | |
| WORKFLOW_URL: ${{ env.WORKFLOW_URL }} | |
| WORKFLOW_FILE: ${{ env.WORKFLOW_FILE }} | |
| COMMIT: ${{ env.COMMIT }} | |
| COMMIT_URL: ${{ env.COMMIT_URL }} | |
| PR_URL: ${{ env.PR_URL }} | |
| with: | |
| filename: .github/SOLIDITY_COMPAT_ISSUE.md | |
| # Finds the last open issue matching given labels | |
| - name: Find the last open compatibility issue | |
| id: last-issue | |
| if: steps.solidity-test.outcome != 'success' && (github.event_name != 'pull_request' || github.event.action == 'enqueued') | |
| uses: micalevisk/last-issue-action@v2 | |
| with: | |
| repository: argumentcomputer/solidity-verifier | |
| state: open | |
| # Find the last updated open issue that has these labels: | |
| labels: | | |
| compatibility | |
| debt | |
| automated issue | |
| # Update existing issue in `solidity-verifier` or create new one | |
| - uses: peter-evans/create-issue-from-file@v5 | |
| if: steps.solidity-test.outcome != 'success' && (github.event_name != 'pull_request' || github.event.action == 'enqueued') | |
| with: | |
| token: ${{ secrets.REPO_TOKEN }} | |
| repository: argumentcomputer/solidity-verifier | |
| issue-number: ${{ steps.last-issue.outputs.issue-number }} | |
| title: ":rotating_light: Arecibo compatibility is broken" | |
| content-filepath: .github/SOLIDITY_COMPAT_ISSUE.md | |
| labels: | | |
| compatibility | |
| debt | |
| automated issue |