Synthesis ~10% regression #30
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Creates a PR benchmark comment with a comparison to main | |
| name: Benchmark pull requests | |
| on: | |
| issue_comment: | |
| types: [created] | |
| env: | |
| CARGO_TERM_COLOR: always | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| cpu-benchmark: | |
| name: run fibonacci benchmark | |
| runs-on: buildjet-32vcpu-ubuntu-2204 | |
| if: | |
| github.event.issue.pull_request | |
| && github.event.issue.state == 'open' | |
| && contains(github.event.comment.body, '!benchmark') | |
| && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Checkout PR branch | |
| run: gh pr checkout $PR_NUMBER | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.issue.number }} | |
| # Install dependencies | |
| - name: Install dependencies | |
| run: sudo apt-get install -y pkg-config libssl-dev | |
| - uses: actions-rs/toolchain@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Load env vars | |
| run: | | |
| set -a | |
| source bench.env | |
| set +a | |
| echo "LURK_BENCH_OUTPUT=pr-comment" >> $GITHUB_ENV | |
| env | grep -E 'LURK|EC_GPU|CUDA' | |
| working-directory: ${{ github.workspace }}/benches | |
| # Run the comparative benchmark and comment output on the PR | |
| - uses: boa-dev/criterion-compare-action@v3 | |
| with: | |
| # Optional. Compare only this benchmark target | |
| benchName: "fibonacci" | |
| # Needed. The name of the branch to compare with | |
| branchName: ${{ github.ref_name }} | |
| gpu-benchmark: | |
| name: run fibonacci benchmark on GPU | |
| runs-on: [self-hosted, gpu-bench] | |
| if: | |
| github.event.issue.pull_request | |
| && github.event.issue.state == 'open' | |
| && contains(github.event.comment.body, '!gpu-benchmark') | |
| && (github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Checkout PR branch | |
| run: gh pr checkout $PR_NUMBER | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.issue.number }} | |
| # Set up GPU | |
| # Check we have access to the machine's Nvidia drivers | |
| - run: nvidia-smi | |
| # The `compute`/`sm` number corresponds to the Nvidia GPU architecture | |
| # In this case, the self-hosted machine uses the Ampere architecture, but we want this to be configurable | |
| # See https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ | |
| # Writes env vars to `bench.env` to be read by `dotenv` action. This is roundabout but puts all the env vars in one place | |
| - name: Set env for CUDA compute | |
| run: echo "CUDA_ARCH=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | sed 's/\.//g')" >> bench.env | |
| - name: set env for EC_GPU | |
| run: echo 'EC_GPU_CUDA_NVCC_ARGS=--fatbin --gpu-architecture=sm_${{ env.CUDA_ARCH }} --generate-code=arch=compute_${{ env.CUDA_ARCH }},code=sm_${{ env.CUDA_ARCH }}' >> bench.env | |
| # Check that CUDA is installed with a driver-compatible version | |
| # This must also be compatible with the GPU architecture, see above link | |
| - run: nvcc --version | |
| # Install dependencies | |
| - uses: actions-rs/toolchain@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Load env vars | |
| run: | | |
| set -a | |
| source bench.env | |
| set +a | |
| echo "LURK_BENCH_OUTPUT=pr-comment" >> $GITHUB_ENV | |
| env | grep -E 'LURK|EC_GPU|CUDA' | |
| working-directory: ${{ github.workspace }}/benches | |
| # Run the comparative benchmark and comment output on the PR | |
| - uses: boa-dev/criterion-compare-action@v3 | |
| with: | |
| # Note: Removing `benchName` causes `criterion` errors: https://github.com/boa-dev/criterion-compare-action#troubleshooting | |
| # Optional. Compare only this benchmark target | |
| benchName: "fibonacci" | |
| # Optional. Features activated in the benchmark | |
| features: "cuda" | |
| # Needed. The name of the branch to compare with | |
| branchName: ${{ github.ref_name }} |