Write tests for core functions

.github/workflows/development.yml

@@ -10,6 +10,9 @@ jobs:
   build-venv:
     runs-on: ubuntu-22.04
     steps:
+      - run: |
+          sudo apt-get upgrade
+          sudo apt-get -y install intel-mkl
       - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v5
@@ -20,11 +23,13 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ./venv
-          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/**.toml') }}
+          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements/base.txt') }}-${{ hashFiles('**/**.toml') }}
       - name: Build venv
         run: |
           python -m pip install --upgrade pip
           python -m venv ./venv
+          # with mkl support
+          ./venv/bin/python -m pip install -r requirements/mkl.txt
           ./venv/bin/python -m pip install .[testing]
         if: steps.cached-venv.outputs.cache-hit != 'true'
       - name: Cache venv
@@ -44,7 +49,7 @@ jobs:
         uses: actions/cache/restore@v4
         with:
           path: ./venv
-          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/**.toml') }}
+          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements/base.txt') }}-${{ hashFiles('**/**.toml') }}
       - name: Build wheel
         run: |
           venv/bin/python -m pip install build
@@ -64,7 +69,7 @@ jobs:
       - uses: actions/cache/restore@v4
         with:
           path: ./venv
-          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/**.toml') }}
+          key: venv-${{ runner.os }}-${{ hashFiles('**/requirements/base.txt') }}-${{ hashFiles('**/**.toml') }}
       - name: Getting build wheel
         uses: actions/download-artifact@v4
         with:
@@ -84,6 +89,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: install
     steps:
+      - run: |
+          sudo apt-get upgrade
+          sudo apt-get -y install intel-mkl
       - uses: actions/checkout@v4
       - name: Set up Python 3.9
         uses: actions/setup-python@v5
@@ -95,9 +103,85 @@ jobs:
         with:
           path: ./venv
           key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }}
-      - name: Test with pytest
+      - name: Tests without memory tracking
         run: |
-          venv/bin/python -m pytest
+          venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 -k 'not _memory'
+        env:
+          COVERAGE_FILE: ".coverage.no_memory"
+      - name: Store coverage file
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-no_memory
+          path: .coverage.no_memory
+          include-hidden-files: true
+
+  test-memory:
+    runs-on: ubuntu-latest
+    needs: install
+    steps:
+      - run: |
+          sudo apt-get upgrade
+          sudo apt-get -y install intel-mkl
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+      - name: Build venv
+        run: |
+          python -m pip install --upgrade pip
+          python -m venv ./venv
+          # with mkl support
+          ./venv/bin/python -m pip install .[testing,mkl]
+      # # Note: we use cache here since artifacts do not keep permissions
+      # - name: Getting the venv
+      #   uses: actions/cache/restore@v4
+      #   with:
+      #     path: ./venv
+      #     key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }}
+      - name: Tests with memory tracking
+        run: |
+          venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray --most-allocations=0 -k '_memory'
+        env:
+          COVERAGE_FILE: ".coverage.memory"
+      - name: Store coverage file
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-memory
+          path: .coverage.memory
+          include-hidden-files: true
+
+  coverage:
+    name: combine-coverage
+    runs-on: ubuntu-22.04
+    needs: [test, test-memory]
+    if: |
+      always()
+    permissions:
+      pull-requests: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        id: download
+        with:
+          pattern: coverage-*
+          merge-multiple: true
+      - name: Coverage comment
+        id: coverage_comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MERGE_COVERAGE_FILES: true
+
+      - name: Store Pull Request comment to be posted
+        uses: actions/upload-artifact@v4
+        if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
+        with:
+          name: python-coverage-comment-action
+          path: python-coverage-comment-action.txt
 
   lint:
     runs-on: ubuntu-latest
@@ -117,3 +201,4 @@ jobs:
       - name: Lint with ruff
         run: |
           venv/bin/python -m ruff check --select=ALL --output-format=github src/
+        continue-on-error: true

.github/workflows/report_coverage.yml

@@ -0,0 +1,29 @@
+name: Post coverage comment
+
+on:
+  workflow_run:
+    workflows: ["Development Workflow"]
+    types:
+      - completed
+
+jobs:
+  test:
+    name: Run tests & display coverage
+    runs-on: ubuntu-22.04
+    # if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    if: github.event.workflow_run.event == 'pull_request'
+    permissions:
+      pull-requests: write
+      contents: write
+      actions: read
+    steps:
+      # DO NOT run actions/checkout here, for security reasons
+      # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+      - name: Post comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
+          # Update those if you changed the default values:
+          # COMMENT_ARTIFACT_NAME: python-coverage-comment-action
+          # COMMENT_FILENAME: python-coverage-comment-action.txt

.readthedocs.yaml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.12"
+    python: "3.9"
 
 # Build documentation in the "docs/" directory with Sphinx
 sphinx:

docs/conf.py

@@ -5,10 +5,8 @@
 import pathlib
 import sys
 
-# -- Add the project root for autodiscovery with sphinx.ext.autodoc ----------
+# -- Add the project root for autodiscovery ----------
 sys.path.insert(0, pathlib.Path(__file__).parents[1].resolve().as_posix())
-autodoc_typehints = 'description'
-autodoc_class_signature = 'separated'
 
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
@@ -44,4 +42,5 @@
 autoapi_dirs = ["../src/", ]
 autoapi_file_patterns = ['*.py', ]
 autoapi_member_order = "groupwise"
+autoapi_python_class_content = "both"
 # autoapi_ignore = ["*conf.py", "*setup.py" , "*_cython*.pyx", ]

pyproject.toml

@@ -24,9 +24,14 @@ Repository = "https://github.com/alexbovet/flow_stability.git"
 Documentation = "https://flow-stability.readthedocs.io/"
 Issues = "https://github.com/alexbovet/flow_stability/issues"
 
-[tool.setuptools.dynamic]
-dependencies = {file = ["requirements.txt"]}
-readme = {file = ["README.md"]}
+[tool.setuptools.dynamic.readme]
+file = ["README.md"]
+
+[tool.setuptools.dynamic.dependencies]
+file = ["requirements/base.txt"]
+
+[tool.setuptools.dynamic.optional-dependencies.mkl]
+file = ["requirements/mkl.txt"]
 
 [tool.setuptools.dynamic.optional-dependencies.testing]
 file = ["tests/requirements.txt"]
@@ -47,6 +52,21 @@ requires = [
 ]
 build-backend = "setuptools.build_meta"
 
+# config for the test coverage
+[tool.coverage.run]
+relative_files = true
+# plugins = ["Cython.Coverage", ]
+source = ["src/flowstab", ]
+omit = [
+  "tests",
+  "*__init__.py",
+  ]
+[tool.coverage.paths]
+source = [
+  "src/flowstab/", 
+  "venv/lib/python*/site-packages/flowstab/"
+  ]
+
 # This is for the linting with ruff
 [tool.ruff]
 target-version = "py39"

requirements.txt → requirements/base.txt

@@ -1,5 +1,5 @@
 Cython>=3.0.11
 pandas>=2.2.2
-scipy>=1.13.1
 hickle>=5.0.3
 psutil>=6.0.0
+scipy>=1.13

requirements/mkl.txt

@@ -0,0 +1 @@
+sparse_dot_mkl>=0.9.4

src/flowstab/FlowStability.py

@@ -1,23 +1,26 @@
 """#
 # flow stability
 #
 # Copyright (C) 2021 Alexandre Bovet <[email protected]>
 #
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU Lesser General Public License as published by the Free
 # Software Foundation; either version 3 of the License, or (at your option) any
 # later version.
 #
 # This program is distributed in the hope that it will be useful, but WITHOUT
 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 # details.
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 
 """
+from __future__ import annotations
+from typing import Collection
+
 import importlib.util
 import os
 import time
@@ -39,13 +42,23 @@
     sparse_matmul,
     sparse_stoch_mat,
 )
-from .TemporalNetwork import inplace_csr_row_normalize, set_to_zeroes, sparse_lapl_expm
+from .TemporalNetwork import (
+    inplace_csr_row_normalize,
+    set_to_zeroes,
+    sparse_lapl_expm
+)
 
 USE_CYTHON = True
 if importlib.util.find_spec("cython") is not None:
-    from _cython_fast_funcs import compute_S, cython_nmi, cython_nvi, sum_Sout, sum_Sto
+    from _cython_fast_funcs import (
+        compute_S,
+        cython_nmi,
+        cython_nvi,
+        sum_Sout,
+        sum_Sto
+    )
 else:
     print("Could not load cython functions")
     USE_CYTHON = False
 
 
@@ -55,10 +68,22 @@
     and a node to cluster dict.
     """
 
-    def __init__(self, num_nodes,
-                 cluster_list=None,
-                 node_to_cluster_dict=None,
-                 check_integrity=False):
+    def __init__(self,
+                 num_nodes:int,
+                 cluster_list:Collection|None=None,
+                 node_to_cluster_dict:dict|None=None,
+                 check_integrity:bool=False):
+        """
+        Parameters
+        ----------
+        num_nodes:
+          The number of nodes in the partition
+        cluster_list:
+          A list of clusters with each cluster being a set of nodes
+        node_to_custer_dict:
+          A mapping that maps each node to the index of the corresponding
+          cluster in `cluster_list`
+        """
 
         self.num_nodes = num_nodes
 
@@ -2360,6 +2385,7 @@
         return Clustering(p1=pi,p2=pi,T=T, rnd_seed=rnd_seed)
 
 
+# TODO: move to helper scripts
 def n_random_seeds(n):
     # generate n random seeds