Merge pull request #215 from NeuroBench/dev

Release 1.0.4

.bumpversion.toml

@@ -0,0 +1,27 @@
+
+[tool.bumpversion]
+parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
+serialize = ["{major}.{minor}.{patch}"]
+regex = false
+current_version = "1.0.4"
+ignore_missing_version = false
+search = "{current_version}"
+replace = "{new_version}"
+tag = false
+sign_tags = false
+tag_name = "{new_version}"
+tag_message = "Bump version: {current_version} → {new_version}"
+allow_dirty = false
+commit = false
+message = "Bump version: {current_version} → {new_version}"
+commit_args = ""
+
+[[tool.bumpversion.files]]
+filename = "pyproject.toml"
+search = "version = \"{current_version}\""
+replace = "version = \"{new_version}\""
+
+[[tool.bumpversion.files]]
+filename = "docs/conf.py"
+search = "release = \"{current_version}\""
+replace = "release = \"{new_version}\""

.github/workflows/publish-distribution.yaml

@@ -0,0 +1,36 @@
+name: Publish Distribution to PyPI
+
+on:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  build-and-publish-final-dist:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        run: |
+          pip install poetry
+
+      - name: Install dependencies
+        run: |
+          poetry install --without dev
+
+      - name: Build the package
+        run: |
+          poetry build
+
+      - name: Publish to PyPI
+        env:
+          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_API_TOKEN }}
+        run: |
+          poetry publish 

.readthedocs.yaml

@@ -10,11 +10,17 @@ build:
       # https://python-poetry.org/docs/#installing-manually
       - pip install poetry
       # Tell poetry to not use a virtual environment
-      - poetry config virtualenvs.create false
+      # - poetry config virtualenvs.create false
     post_install:
       # Install dependencies with 'docs' dependency group
       # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
-      - poetry install --with dev
+      # VIRTUAL_ENV needs to be set manually for now.
+      # See https://github.com/readthedocs/readthedocs.org/pull/11152/
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with dev
 
 sphinx:
-  configuration: docs/conf.py
+  configuration: docs/conf.py
+
+# python:
+#   install:
+#     - requirements: docs/requirements.txt

RELEASE.rst

@@ -0,0 +1,70 @@
+Version Release Guidelines
+=======================
+
+This document describes the guidelines for releasing new versions of the library. We follow semantic versioning, which means our version numbers have three parts: MAJOR.MINOR.PATCH.
+
+- MAJOR version when you make incompatible API changes
+- MINOR version when you add functionality in a backwards-compatible manner
+- PATCH version when you make backwards-compatible bug fixes
+
+
+1. Install the `bump-my-version` package:
+
+    ```
+    pip install --upgrade bump-my-version
+    ```
+--------------------
+
+2.  Create a new branch for the release from dev branch:
+
+    ```
+    git checkout -b release/x.y.z
+    ```
+--------------------
+
+3. Update the version number using the `bump-my-version` command:
+
+    ```
+    bump-my-version bump path
+    ```
+    or
+    ```
+    bump-my-version bump minor
+    ```
+    or
+    ```
+    bump-my-version bump major
+    ```
+--------------------
+
+4. Commit the changes with the following message and push the changes to the release branch:
+
+    ```
+    git commit -m "Bump version: {current_version} → {new_version}"
+    ```
+
+    ```
+    git push origin release/x.y.z
+    ```
+
+--------------------
+
+5. Create a pull request from the release branch to the dev branch.
+
+6. Once the pull request is approved and merged, create a new pull request from the dev branch to the master branch.
+
+7. Once the pull request is approved and merged, create the tag on the main branch to invoke the package publishing workflow:
+
+    ```
+    git tag -a x.y.z -m "Release x.y.z"
+    ```
+
+    ```
+    git push origin tag <tag_name>
+    ```
+--------------------
+
+8. Once the tag is pushed, the package publishing workflow will be triggered and the package will be published to the PyPI.
+
+9. Once the package is published, create a new release on GitHub with the tag name and the release notes (generate them automatically).
+

docs/conf.py

@@ -14,7 +14,7 @@
 project = "NeuroBench"
 copyright = "2024, Jason Yik, Noah Pacik-Nelson, Korneel Van Den Berghe"
 author = "Jason Yik, Noah Pacik-Nelson, Korneel Van Den Berghe"
-release = "v1.0.0"
+release = "1.0.4"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

docs/neurobench.benchmarks.rst

@@ -4,8 +4,8 @@ neurobench.benchmarks
 Benchmark
 ^^^^^^^^^
 
-.. automodule:: neurobench.benchmarks.benchmark
-    :members:
+.. automodule:: neurobench.benchmarks
+    :members: Benchmark
     :undoc-members:
     :show-inheritance:
 

docs/neurobench.datasets.rst

@@ -15,21 +15,6 @@ as linear 16-bit, single-channel, pulse code modulated values, at a 16 kHz sampl
     :show-inheritance:
 
 
-DVS Gestures
-^^^^^^^^^^^^
-
-The IBM Dynamic Vision Sensor (DVS) Gesture dataset is composed of recordings of 29 distinct individuals executing 10 different
-types of gestures, including but not limited to clapping, waving, etc. Additionally, an 11th gesture class is included that comprises 
-gestures that cannot be categorized within the first 10 classes. The gestures are recorded under four distinct lighting conditions, 
-and each gesture is associated with a label that indicates the corresponding lighting condition under which it was performed.
-
-.. automodule:: neurobench.datasets.dvs_gesture
-    :special-members: __init__, __getitem__
-    :members:
-    :undoc-members:
-    :show-inheritance:
-
-
 Prophesee Megapixel Automotive
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx-rtd-theme
+tqdm
+tonic
+numpy

docs/tutorial/index.rst

@@ -52,7 +52,10 @@ using snnTorch.
         snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True),
     )
 
-To get started, we will load our desired dataset in a dataloader:
+To get started, we will load our desired dataset in a dataloader. Note that any
+torch.Dataloader can be used for the benchmark, it is not constrained to the datasets
+available in the harness. Check out the `Tonic library <https://tonic.readthedocs.io/en/latest/#>`_
+for an excellent resource for neuromorphic datasets!
 
 .. code:: python
 

neurobench/benchmarks/__init__.py

@@ -1 +1 @@
-from .benchmark import *
+from .benchmark import Benchmark

neurobench/benchmarks/benchmark.py

@@ -5,7 +5,12 @@
 from . import static_metrics, workload_metrics
 
 # workload metrics which require hooks
-requires_hooks = ["activation_sparsity", "number_neuron_updates", "synaptic_operations"]
+requires_hooks = [
+    "activation_sparsity",
+    "number_neuron_updates",
+    "synaptic_operations",
+    "membrane_updates",
+]
 
 
 class Benchmark:

neurobench/benchmarks/hooks.py

@@ -21,6 +21,8 @@ def __init__(self, layer, connection_layer=None, prev_act_layer_hook=None):
         """
         self.activation_outputs = []
         self.activation_inputs = []
+        self.pre_fire_mem_potential = []
+        self.post_fire_mem_potential = []
         if layer is not None:
             self.hook = layer.register_forward_hook(self.hook_fn)
             self.hook_pre = layer.register_forward_pre_hook(self.pre_hook_fn)
@@ -46,6 +48,8 @@ def pre_hook_fn(self, layer, input):
 
         """
         self.activation_inputs.append(input)
+        if self.spiking:
+            self.pre_fire_mem_potential.append(layer.mem)
 
     def hook_fn(self, layer, input, output):
         """
@@ -62,6 +66,7 @@ def hook_fn(self, layer, input, output):
         """
         if self.spiking:
             self.activation_outputs.append(output[0])
+            self.post_fire_mem_potential.append(layer.mem)
 
         else:
             self.activation_outputs.append(output)
@@ -75,6 +80,8 @@ def reset(self):
         """Resets the stored activation outputs and inputs."""
         self.activation_outputs = []
         self.activation_inputs = []
+        self.pre_fire_mem_potential = []
+        self.post_fire_mem_potential = []
 
     def close(self):
         """Remove the registered hook."""

neurobench/benchmarks/workload_metrics.py

@@ -2,6 +2,7 @@
 import numpy as np
 from ..utils import check_shape, make_binary_copy, single_layer_MACs
 from .hooks import ActivationHook, LayerHook
+from collections import defaultdict
 
 
 class AccumulatedMetric:
@@ -121,6 +122,68 @@ def activation_sparsity(model, preds, data):
     return sparsity
 
 
+class membrane_updates(AccumulatedMetric):
+    """
+    Number of membrane potential updates.
+
+    This metric can only be used for spiking models implemented with SNNTorch.
+
+    """
+
+    def __init__(self):
+        """Init metric state."""
+        self.total_samples = 0
+        self.neuron_membrane_updates = defaultdict(int)
+
+    def reset(self):
+        """Reset metric state."""
+        self.total_samples = 0
+        self.neuron_membrane_updates = defaultdict(int)
+
+    def __call__(self, model, preds, data):
+        """
+        Number of membrane updates of the model forward.
+
+        Args:
+            model: A NeuroBenchModel.
+            preds: A tensor of model predictions.
+            data: A tuple of data and labels.
+        Returns:
+            float: Number of membrane potential updates.
+
+        """
+        for hook in model.activation_hooks:
+            for index_mem in range(len(hook.pre_fire_mem_potential) - 1):
+                pre_fire_mem = hook.pre_fire_mem_potential[index_mem + 1]
+                post_fire_mem = hook.post_fire_mem_potential[index_mem + 1]
+                nr_updates = torch.count_nonzero(pre_fire_mem - post_fire_mem)
+                self.neuron_membrane_updates[str(type(hook.layer))] += int(nr_updates)
+            self.neuron_membrane_updates[str(type(hook.layer))] += int(
+                torch.numel(hook.post_fire_mem_potential[0])
+            )
+        self.total_samples += data[0].size(0)
+        return self.compute()
+
+    def compute(self):
+        """
+        Compute membrane updates using accumulated data.
+
+        Returns:
+            float: Compute the total updates to each neuron's membrane potential within the model,
+            aggregated across all neurons and normalized by the number of samples processed.
+
+        """
+        if self.total_samples == 0:
+            return 0
+
+        total_mem_updates = 0
+        for key in self.neuron_membrane_updates:
+            total_mem_updates += self.neuron_membrane_updates[key]
+
+        total_updates_per_sample = total_mem_updates / self.total_samples
+        return total_updates_per_sample
+
+
 def number_neuron_updates(model, preds, data):
     """
     Number of times each neuron type is updated.

neurobench/datasets/MSWC_dataset.py

@@ -257,6 +257,7 @@ class MSWC(Dataset):
     Subset version (https://huggingface.co/datasets/NeuroBench/mswc_fscil_subset)
     of the original MSWC dataset (https://mlcommons.org/en/multilingual-spoken-words/)
     for a few-shot class-incremental learning (FSCIL) task consisting of 200 voice commands keywords:
+
     - 100 base classes available for pre-training with:
         - 500 train samples
         - 100 validation samples

neurobench/datasets/__init__.py

@@ -21,12 +21,6 @@ def Gen4DetectionDataLoader(*args, **kwargs):
     )(*args, **kwargs)
 
 
-def DVSGesture(*args, **kwargs):
-    return _lazy_import("neurobench.datasets", ".dvs_gesture", "DVSGesture")(
-        *args, **kwargs
-    )
-
-
 def MackeyGlass(*args, **kwargs):
     return _lazy_import("neurobench.datasets", ".mackey_glass", "MackeyGlass")(
         *args, **kwargs