Codspeed setup

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: codspeed-benchmarks
+
+on:
+  # Run on pushes to the main branch
+  push:
+    branches:
+      - "master" # or "main"
+  # Run on pull requests
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      # ...
+      # Setup your environment here:
+      #  - Configure your Python/Rust/Node version
+      #  - Install your dependencies
+      #  - Build your benchmarks (if using a compiled language)
+      # ...
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: false
+          miniconda-version: latest
+
+      - name: Create Conda environment
+        run: conda env create -n bchmark 
+
+      - name: Activate Conda environment
+        run: conda activate bchmark
+
+      - name: Install project dependencies
+        run: pip install -e . && pip install pytest
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          token: ${{ secrets.codspeed_token }}
+          run: "pytest tests/kernels/test_layernorm.py"

tests/kernels/test_layernorm.py

@@ -10,6 +10,7 @@
 SEEDS = [0]
 
 
+@pytest.mark.benchmark
 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
 @pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
 @pytest.mark.parametrize("add_residual", ADD_RESIDUAL)