WIP Start adding some basic benchmarking infrastructure

.github/workflows/benchmark-main.yml

@@ -0,0 +1,42 @@
+name: Run benchmarks for main
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+  schedule:
+    #        ┌───────── minute (0 - 59)
+    #        │ ┌───────── hour (0 - 23)
+    #        │ │ ┌───────── day of the month (1 - 31)
+    #        │ │ │ ┌───────── month (1 - 12 or JAN-DEC)
+    #        │ │ │ │ ┌───────── day of the week (0 - 6 or SUN-SAT)
+    - cron: '0 9 * * 1'  # Every Monday at 0900 UTC
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@main
+    container:
+      volumes:
+        - ~:/github
+    with:
+      default_python: '3.10'
+      envs: |
+        - linux: benchmark-main
+
+  update-repo:
+    runs-on: ubuntu-latest
+    container:
+      volumes:
+        - ~:/github
+    needs: [benchmarks]
+    run: |
+      git clone https://github.com/DKISTDC/dkist-benchmarks.git /github/dkist-benchmarks
+      cd /dkist-benchmarks
+      git add main.json
+      git commit -m "Update benchmark data for main"
+      git push origin main

.github/workflows/main.yml

@@ -42,6 +42,7 @@ jobs:
         - windows: py311-online
         - macos: py310
         - linux: py310-oldestdeps
+        - linux: benchmarks
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

dkist/tests/test_benchmarks.py

@@ -0,0 +1,17 @@
+import gzip
+from pathlib import Path
+
+from dkist import load_dataset
+from dkist.data.test import rootdir
+
+
+def load_asdf(vispdir):
+    ds = load_dataset(vispdir / "test_visp.asdf")
+
+
+def test_load_asdf(benchmark, tmp_path_factory):
+    vispdir = tmp_path_factory.mktemp("data")
+    with gzip.open(Path(rootdir) / "large_visp.asdf.gz", mode="rb") as gfo:
+        with open(vispdir / "test_visp.asdf", mode="wb") as afo:
+            afo.write(gfo.read())
+    benchmark(load_asdf, vispdir)

pyproject.toml

@@ -53,6 +53,7 @@ tests = [
   "pytest-mpl",
   "pytest-httpserver",
   "pytest-filter-subpackage",
+  "pytest-benchmark",
   "hypothesis",
   "tox",
   "pydot",

tox.ini

@@ -10,6 +10,7 @@ envlist =
     py310-oldestdeps
     build_docs{,-notebooks}
     codestyle
+    benchmarks
 
 [testenv]
 pypi_filter = https://raw.githubusercontent.com/sunpy/sunpy/main/.test_package_pins.txt
@@ -89,3 +90,17 @@ commands =
     !notebooks: sphinx-build -j 1 --color -W --keep-going -b html -d _build/.doctrees . _build/html -D nb_execution_mode=off {posargs}
     notebooks: sphinx-build -j 1 --color -W --keep-going -b html -d _build/.doctrees . _build/html {posargs}
     python -c 'import pathlib; print("Documentation available under file://\{0\}".format(pathlib.Path(r"{toxinidir}") / "docs" / "_build" / "index.html"))'
+
+[testenv:benchmarks]
+description = Run benchmarks on PR and compare against main to ensure there are no performance regressions
+allowlist_externals=git
+commands =
+    git clone https://github.com/DKISTDC/dkist-benchmarks.git {env_tmp_dir}/dkist-benchmarks
+    {env:PYTEST_COMMAND} -k test_benchmarks --benchmark-autosave --benchmark-compare={env_tmp_dir}/dkist-benchmarks/main.json --benchmark-compare-fail=mean:5%
+
+[testenv:benchmark-main]
+description = Run benchmarks on main to keep comparison data up to date
+allowlist_externals = git,cp
+commands =
+    {env:PYTEST_COMMAND} -k test_benchmarks --benchmark-save=main
+    cp {toxinidir}/.tmp/benchmark-main/.benchmarks/Linux-CPython-3.10-64bit/0001_main.json /github/main.json