benchmark.yml

name: Benchmark
on:
  pull_request:
    paths:
      - '.github/workflows/benchmark.yml'
      - 'src/**'
      - 'tests/**'

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse

jobs:
  benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
        timeout-minutes: 3
        with:
          persist-credentials: false
          path: "cryptography-pr"
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
        timeout-minutes: 3
        with:
          repository: "pyca/cryptography"
          path: "cryptography-base"
          ref: "${{ github.base_ref }}"

      - name: Setup python
        id: setup-python
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
        with:
          python-version: "3.11"

      - name: Create virtualenv (base)
        run: |
          python -m venv .venv-base
          .venv-base/bin/pip install -v -c ./cryptography-base/ci-constraints-requirements.txt "./cryptography-base[test]" ./cryptography-base/vectors/
      - name: Create virtualenv (PR)
        run: |
          python -m venv .venv-pr
          .venv-pr/bin/pip install -v -c ./cryptography-pr/ci-constraints-requirements.txt "./cryptography-pr[test]" ./cryptography-pr/vectors/

      - name: Run benchmarks (base)
        run: .venv-base/bin/pytest --benchmark-enable --benchmark-only ./cryptography-pr/tests/bench/ --benchmark-json=bench-base.json
      - name: Run benchmarks (PR)
        run: .venv-pr/bin/pytest --benchmark-enable --benchmark-only ./cryptography-pr/tests/bench/ --benchmark-json=bench-pr.json

      - name: Compare results
        run: python ./cryptography-pr/.github/compare_benchmarks.py bench-base.json bench-pr.json | tee -a $GITHUB_STEP_SUMMARY