benchmark.yaml

name: Benchmark
on:
  push:
    branches:
      - master
jobs:
  benchmarks:
    name: "Benchmark time & memory"
    runs-on: "ubuntu-latest"
    steps:
      - uses: actions/checkout@v4
      - name: Set Up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
      - name: Update pip
        run: python -m pip install --upgrade pip
      - name: Install dependencies
        run: |
          python -m pip install -e .[test,benchmark,docs]
      - name: Run Benchmark for time
        run: pytest --benchmark-json output.json -k _time tests/benchmarks

      - name: Download previous benchmark data
        uses: actions/cache@v3
        with:
          path: ./cache
          key: ${{ runner.os }}-benchmark
      - name: Extract benchmark data
        run:  python .github/scripts/extract_benchmark_data.py output.json output_action.json
      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          # What benchmark tool the output.txt came from
          tool: 'customSmallerIsBetter'
          # Where the output from the benchmark tool is stored
          output-file-path: output_action.json
          # Where the previous data file is stored
          # external-data-json-path: ./cache/benchmark-data.json
          # Workflow will fail when an alert happens
          fail-on-alert: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          alert-threshold: '150%'
          comment-on-alert: true
          comment-always: false
          auto-push: true
          gh-pages-branch: "benchmarks"
          benchmark-data-dir-path: "docs/bench"
          alert-comment-cc-users: '@danwos'

      - name: Run Benchmark for memory
        run: pytest --benchmark-json output.json -k _memory tests/benchmarks
      - name: Create memory flamegraph
        run: memray flamegraph -o mem_out.html mem_out.bin
      - name: Publish memory flamegraph
        run: |
          git checkout benchmarks
          cp mem_out.html docs/memory.html
          git add docs/memory.html
          git -c "user.name=Sphinx-Needs CI" -c "user.email=ci@sphinx-needs.com" commit -m "memory.html update"
      - name: Push changes   
        run: git push origin benchmarks