benchmarks.yml

name: Benchmarks

on:
  workflow_call:
    inputs:
      llvm_build_type:
        type: string
        description: "LLVM build type: debug | release"
        required: false
        default: "release"
      compiler_tester_reference_branch:
        type: string
        description: "compiler-tester branch to use as a benchmark reference"
        required: false
        default: "main"
      compiler_tester_candidate_branch:
        type: string
        description: "compiler-tester branch to use as a benchmark candidate"
        required: false
        default: "main"
      compiler_llvm_reference_branch:
        type: string
        description: "compiler-llvm branch to use as a benchmark reference"
        required: false
        default: "main"
      compiler_llvm_candidate_branch:
        type: string
        description: "compiler-llvm branch to use as a benchmark candidate"
        required: false
        default: "main"
      compiler_llvm_benchmark_mode:
        type: string
        description: "Mode filter for compiler-llvm benchmarks"
        required: false
        default: "^M^B3"
      compiler_llvm_benchmark_path:
        type: string
        description: "Path filter for compiler-llvm benchmarks"
        required: false
        default: ""
      ccache-key-type:
        type: string
        required: false
        default: 'dynamic'
        description: 'Type of ccache key: `static` - rotate each month, or `dynamic` - rotate with each update of `LLVM.lock`.'
      compiler-tester-repo:
        type: string
        required: false
        default: 'matter-labs/era-compiler-tester'
        description: 'Compiler tester repository to use. Required for forks testing.'
      compiler-llvm-repo:
        type: string
        required: false
        default: 'matter-labs/era-compiler-llvm'
        description: 'Compiler LLVM repository to use. Required for forks testing.'
      custom-solc-run-id:
        type: string
        required: false
        default: ''
        description: 'run id of custom zksync solc artifact to download.'
      custom-solc-version:
        type: string
        required: false
        default: ''
        description: 'custom solc version to use for benchmarks'
      target-machine:
        type: string
        required: false
        default: 'eravm'
        description: 'Target machine passed via `--target` for era-compiler-tester. Available arguments: `eravm`, `evm`.'

jobs:

  benchmarks:
    strategy:
      fail-fast: false
      matrix:
        type: ["reference", "candidate"]
    name: ${{ inputs.target-machine}} ${{ matrix.type }}
    runs-on: matterlabs-ci-runner-high-performance
    container:
      image: ghcr.io/matter-labs/zksync-llvm-runner:latest
      options: -m 110g
    steps:
      - name: Define branches
        shell: bash -ex {0}
        id: define-branches
        run: |
          if [ "${{ matrix.type }}" = "candidate" ]; then
            echo "compiler-tester-branch=${{ inputs.compiler_tester_candidate_branch || github.head_ref }}" | tee -a "${GITHUB_OUTPUT}"
            echo "llvm-branch=${{ inputs.compiler_llvm_candidate_branch || '' }}" | tee -a "${GITHUB_OUTPUT}"
            echo "compiler-tester-repo=${{ inputs.compiler-tester-repo || github.event.pull_request.head.repo.full_name }}" | tee -a "${GITHUB_OUTPUT}"
          else
            echo "compiler-tester-branch=${{ inputs.compiler_tester_reference_branch || github.event.repository.default_branch }}" | tee -a "${GITHUB_OUTPUT}"
            echo "llvm-branch=${{ inputs.compiler_llvm_reference_branch || '' }}" | tee -a "${GITHUB_OUTPUT}"
            # Always use base repo for the reference testing
            echo "compiler-tester-repo=matter-labs/era-compiler-tester" | tee -a "${GITHUB_OUTPUT}"
          fi

      - name: Checkout compiler-tester
        uses: actions/checkout@v4
        with:
          repository: ${{ steps.define-branches.outputs.compiler-tester-repo }}
          ref: ${{ steps.define-branches.outputs.compiler-tester-branch }}
          submodules: recursive

      - name: Checkout LLVM
        if: steps.define-branches.outputs.llvm-branch != ''
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.compiler-llvm-repo }}
          ref: ${{ steps.define-branches.outputs.llvm-branch }}
          path: llvm

      # An issue prevents to correctly use the same version of composite actions from `workflow_call`
      # https://github.com/actions/toolkit/issues/1264
      # for now, it will always be taken from the latest main
      - name: Build LLVM
        uses: matter-labs/era-compiler-ci/.github/actions/build-llvm@main
        with:
          clone-llvm: ${{ steps.define-branches.outputs.llvm-branch == '' && 'true' || 'false' }}
          enable-assertions: true
          ccache-key-type: ${{ inputs.ccache-key-type }}

      - name: Build compiler-tester
        run: cargo build --verbose --release --bin 'compiler-tester'

      - name: Build compilers
        env:
          CARGO_CHECKOUT_DIR: /usr/local/cargo/git/checkouts
        run: |
          cargo build --verbose --release \
            --manifest-path ${CARGO_CHECKOUT_DIR}/era-compiler-solidity-*/*/Cargo.toml \
            --target-dir './target-zksolc/'
          cargo build --verbose --release \
            --manifest-path ${CARGO_CHECKOUT_DIR}/era-compiler-vyper-*/*/Cargo.toml \
            --target-dir './target-zkvyper/'

      - name: Download custom solc
        if: inputs.custom-solc-run-id != '' && inputs.custom-solc-version != ''
        uses: actions/download-artifact@v4
        with:
          pattern: solc*
          path: ./solc-custom-bin/
          run-id: ${{ inputs.custom-solc-run-id }}
          merge-multiple: true

      - name: Prepare custom solc
        if: inputs.custom-solc-run-id != '' && inputs.custom-solc-version != ''
        run: |
          mkdir -p ./solc-bin
          chmod a+x "./solc-custom-bin/solc-${{ inputs.custom-solc-version}}-${{ matrix.type }}"
          mv "./solc-custom-bin/solc-${{ inputs.custom-solc-version}}-${{ matrix.type }}" \
            "./solc-bin/solc-${{ inputs.custom-solc-version}}"
          echo "Testing with custom solc from run ${{ inputs.custom-solc-run-id }}"
          echo $(./solc-bin/solc-${{ inputs.custom-solc-version}} --version)

      - name: Run benchmarks
        shell: bash -ex {0}
        run: |
          MODE="${{ inputs.compiler_llvm_benchmark_mode || '^M^B3' }}"
          if [[ -n "${{ inputs.target-machine }}" && "${{ inputs.target-machine }}" != "default" ]]; then
            TARGET="--target ${{ inputs.target-machine }}"
            # Always use upstream solc for EVM target for now
            if [[ "${{ inputs.target-machine }}" == "evm" ]]; then
              TOOLCHAIN="--toolchain solc"
              MODE="^"
            fi
          fi
          ./target/release/compiler-tester ${TARGET} ${TOOLCHAIN} \
            --zksolc './target-zksolc/release/zksolc' \
            --zkvyper './target-zkvyper/release/zkvyper' \
            --path=${{ inputs.compiler_llvm_benchmark_path || '' }} \
            --mode="${MODE}" \
            --benchmark=${{ inputs.target-machine }}-${{ matrix.type }}.json

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: compiler-llvm-${{ inputs.target-machine }}-${{ matrix.type }}-benchmark
          path: ${{ inputs.target-machine }}-${{ matrix.type }}.json

  analysis:
    name: "Benchmark comparison"
    runs-on: matterlabs-ci-runner-high-performance
    permissions:
      pull-requests: write
    container:
      image: ghcr.io/matter-labs/zksync-llvm-runner:latest
    needs: benchmarks
    if: ${{ (failure() || success()) && inputs.target-machine != 'evm' }}
    steps:
      - name: Checking out the compiler-tester repository
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.compiler-tester-repo }}
          ref: ${{ inputs.compiler_tester_candidate_branch || 'main' }}
          submodules: recursive

      - uses: actions/download-artifact@v4
        with:
          pattern: compiler-llvm-${{ inputs.target-machine }}*
          merge-multiple: true

      - name: Comparing the LLVM framework benchmark results
        run: |
          cargo run --release --bin benchmark-analyzer -- \
            --reference ${{ inputs.target-machine }}-reference.json \
            --candidate ${{ inputs.target-machine }}-candidate.json \
            --output-file result.txt

      - name: Posting the LLVM benchmark results to the summary
        run: |
          echo '```' | tee -a $GITHUB_STEP_SUMMARY
          cat result.txt | tee -a $GITHUB_STEP_SUMMARY
          echo '```' | tee -a $GITHUB_STEP_SUMMARY
          cat $GITHUB_STEP_SUMMARY > result.txt

      - name: Posting the LLVM benchmark results to a PR comment
        if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork
        uses: mshick/add-pr-comment@v2
        with:
          message-path: result.txt