From 1ab999fa3d9d18b3295561c56b13479a6906c61c Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 19 Aug 2024 15:41:49 +0200 Subject: [PATCH 01/86] adding code coverage --- .github/workflows/development.yml | 2 +- tests/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 77ca6c7..8d03506 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Test with pytest run: | - venv/bin/python -m pytest + venv/bin/python -m pytest --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html lint: runs-on: ubuntu-latest diff --git a/tests/requirements.txt b/tests/requirements.txt index deb66e2..871f1cb 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,3 @@ pytest>=8.3.2 ruff>=0.5.7 +pytest-cov==5.0.0 From 4d219ddc3d1efc132215967fdb8bc5c1a58b915b Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 15:45:32 +0200 Subject: [PATCH 02/86] started SSM init tests --- tests/conftest.py | 13 +++++++++++++ tests/test_sparce_stoch_matrix.py | 20 +++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d920fe7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +import pytest + +import numpy as np +from scipy.sparse import csr_matrix + +@pytest.fixture(scope='function') +def get_csr_matrix(): + """Creat an exemplary csr matrix that can be used for testing + """ + row = np.array([0, 0, 1, 2, 3, 3]) + col = np.array([0, 2, 2, 1, 3, 4]) + data = np.array([1, 2, 3, 5, 1, 6]) + return csr_matrix((data, (row, col)), shape=(5, 5)) diff --git a/tests/test_sparce_stoch_matrix.py b/tests/test_sparce_stoch_matrix.py index d6466cf..4359f83 100644 --- a/tests/test_sparce_stoch_matrix.py +++ b/tests/test_sparce_stoch_matrix.py @@ -1,8 +1,22 @@ -def test_SSM(): +import numpy as np +from scipy.sparse import csr_matrix, coo_matrix + +def test_SSM(get_csr_matrix): """Bacic operations with the 'spares_stoch_mat' class """ - from flowstab.SparseStochMat import sparse_stoch_mat - # TODO + from flowstab.SparseStochMat import sparse_stoch_mat as SSM + # Inits + # ### + # inti from scipy.sparse.csr_matrix + A_csr = get_csr_matrix.copy() + # print(A_csr.toarray()) + ssm = SSM.from_full_csr_matrix(A_csr) + # print(ssm.toarray()) + + + + + def test_SPA(): """Basic operations with the `SpasreStochMat.SPA` class""" From b092cd07796306a316f30508d7dddb884f0820c4 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 15:46:30 +0200 Subject: [PATCH 03/86] silence linter --- .github/workflows/development.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 8d03506..3868ddd 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -117,3 +117,4 @@ jobs: - name: Lint with ruff run: | venv/bin/python -m ruff check --select=ALL --output-format=github src/ + continue-on-error: true From f9d0f18f21be0666643880f01657017815c6c3aa Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 15:53:49 +0200 Subject: [PATCH 04/86] fixing wrong testing command --- .github/workflows/development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 3868ddd..e85d6fe 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Test with pytest run: | - venv/bin/python -m pytest --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html lint: runs-on: ubuntu-latest From 97eadb2a3400b40b2cc2dc24028c36a9276afabb Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 16:31:35 +0200 Subject: [PATCH 05/86] auto-convert to float if needed --- src/flowstab/SparseStochMat.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index ec4fa6e..793e0ad 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -131,14 +131,20 @@ def from_full_csr_matrix(cls, Tcsr, nz_rowcols=None, diag_val=1.0): if nz_rowcols is None: nz_rows, nz_cols = (Tcsr - diag_val * eye(Tcsr.shape[0], format="csr")).nonzero() - nz_rowcols = np.union1d(nz_rows,nz_cols) + nz_rowcols = np.union1d(nz_rows, nz_cols) + + # Make sure we work with floats + Tcsr_data = Tcsr.data.astype(np.float64) + if USE_CYTHON: - res = cython_sparse_stoch_from_full_csr(np.array(nz_rowcols, dtype=np.int32), - Tcsr.data, - Tcsr.indices, - Tcsr.indptr, - diag_val) + res = cython_sparse_stoch_from_full_csr( + np.array(nz_rowcols, dtype=np.int32), + Tcsr_data, + Tcsr.indices, + Tcsr.indptr, + diag_val + ) return cls(*res) @@ -158,7 +164,7 @@ def from_full_csr_matrix(cls, Tcsr, nz_rowcols=None, diag_val=1.0): for tsrow, tbrow in enumerate(nz_rowcols): nzr = 0 for k in range(Tcsr.indptr[tbrow],Tcsr.indptr[tbrow+1]): - T_s_data[its] = Tcsr.data[k] + T_s_data[its] = Tcsr_data[k] T_s_indices[its] = BtoS[Tcsr.indices[k]] its += 1 nzr += 1 From 9340d057c57935f594072e08897db98ebd3e1ecb Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 16:41:15 +0200 Subject: [PATCH 06/86] corrected filename --- ...test_sparce_stoch_matrix.py => test_sparse_stoch_matrix.py} | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) rename tests/{test_sparce_stoch_matrix.py => test_sparse_stoch_matrix.py} (87%) diff --git a/tests/test_sparce_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py similarity index 87% rename from tests/test_sparce_stoch_matrix.py rename to tests/test_sparse_stoch_matrix.py index 4359f83..6585cc2 100644 --- a/tests/test_sparce_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -9,9 +9,8 @@ def test_SSM(get_csr_matrix): # ### # inti from scipy.sparse.csr_matrix A_csr = get_csr_matrix.copy() - # print(A_csr.toarray()) ssm = SSM.from_full_csr_matrix(A_csr) - # print(ssm.toarray()) + np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) From 08e8e8e883315943a815f8920608992721869e41 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 18:46:48 +0200 Subject: [PATCH 07/86] show memory usage and timeing --- .github/workflows/development.yml | 2 +- tests/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index e85d6fe..0dedc13 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Test with pytest run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 --memray lint: runs-on: ubuntu-latest diff --git a/tests/requirements.txt b/tests/requirements.txt index 871f1cb..86d8246 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ pytest>=8.3.2 ruff>=0.5.7 pytest-cov==5.0.0 +pytest-memray==1.7.0 From 4cad73af655c40a646ebe25363b28560450e060b Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 23:07:03 +0200 Subject: [PATCH 08/86] typing --- src/flowstab/SparseStochMat.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 793e0ad..4234a65 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -18,6 +18,8 @@ """ +from __future__ import annotations + import importlib.util import os import time @@ -25,6 +27,7 @@ from functools import wraps import numpy as np +from numpy.typing import NDArray from scipy.sparse import ( coo_matrix, csr_matrix, @@ -113,7 +116,8 @@ def __init__(self, size, data, indices, indptr, nz_rowcols, @classmethod - def from_small_csr_matrix(cls, size, T_small, nz_rowcols, diag_val=1.0): + def from_small_csr_matrix(cls, size:int, T_small:csr_matrix, nz_rowcols:NDArray, + diag_val:float=1.0): """Initialize sparse_stoch_mat from a small csr_matrix""" if not isspmatrix_csr(T_small): raise TypeError("T_small must be in CSR format.") @@ -122,7 +126,8 @@ def from_small_csr_matrix(cls, size, T_small, nz_rowcols, diag_val=1.0): nz_rowcols, diag_val=diag_val) @classmethod - def from_full_csr_matrix(cls, Tcsr, nz_rowcols=None, diag_val=1.0): + def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, + diag_val:float=1.0): """Initialize sparse_stoch_mat from a full size row stochastic csr_matrix """ From a350bfd6bf64fc65b5b32f4944f3f0b3601a8145 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 20 Aug 2024 23:37:04 +0200 Subject: [PATCH 09/86] speed and memory checks --- tests/conftest.py | 14 +++++++++++++- tests/test_sparse_stoch_matrix.py | 29 +++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d920fe7..80561d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,10 +4,22 @@ from scipy.sparse import csr_matrix @pytest.fixture(scope='function') -def get_csr_matrix(): +def get_csr_matrix_small(): """Creat an exemplary csr matrix that can be used for testing """ row = np.array([0, 0, 1, 2, 3, 3]) col = np.array([0, 2, 2, 1, 3, 4]) data = np.array([1, 2, 3, 5, 1, 6]) return csr_matrix((data, (row, col)), shape=(5, 5)) + +@pytest.fixture(scope='function') +def get_csr_matrix_large(): + """Creat an exemplary csr matrix that can be used for testing + """ + size = 10000 + nbr_non_zeros = 1000 + row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + data = np.random.randint(0,100, size=nbr_non_zeros) + density = nbr_non_zeros / size + return csr_matrix((data, (row, col)), shape=(size, size)), density diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 6585cc2..78fed4d 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -1,19 +1,36 @@ import numpy as np -from scipy.sparse import csr_matrix, coo_matrix +import tracemalloc -def test_SSM(get_csr_matrix): +def test_SSM_small(get_csr_matrix_small): """Bacic operations with the 'spares_stoch_mat' class """ from flowstab.SparseStochMat import sparse_stoch_mat as SSM # Inits # ### # inti from scipy.sparse.csr_matrix - A_csr = get_csr_matrix.copy() - ssm = SSM.from_full_csr_matrix(A_csr) - np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) - + A_csr = get_csr_matrix_small + for i in range(10000): + ssm = SSM.from_full_csr_matrix(A_csr) + np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) +def test_SSM_large(get_csr_matrix_large): + """Make sure an SSM does not get expanded during creation + """ + from flowstab.SparseStochMat import sparse_stoch_mat as SSM + # Inits + # ### + # inti from scipy.sparse.csr_matrix + A_csr, density = get_csr_matrix_large + tracemalloc.start() + for i in range(100): + _ = SSM.from_full_csr_matrix(A_csr) + first_size, first_peak = tracemalloc.get_traced_memory() + for i in range(100): + _ = SSM.from_full_csr_matrix(A_csr).toarray() + second_size, second_peak = tracemalloc.get_traced_memory() + assert first_size < density * second_size + assert first_peak < density * second_peak From bc378a78f27fe18bc2db22b81dda24c050b7583e Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 21 Aug 2024 11:22:06 +0200 Subject: [PATCH 10/86] adding details about the ssm --- src/flowstab/SparseStochMat.py | 74 +++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 4234a65..2e49043 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -103,6 +103,47 @@ class sparse_stoch_mat: def __init__(self, size, data, indices, indptr, nz_rowcols, diag_val=1.0): + """Initialize sparse_stoch_mat + + The sparse_stoch_mat will be a square matrix of size `size` with + row/columns of a diagonal matrix for every row/column index not present + in `nz_rowcols`. + For the row/column indices present in `nz_rowcols` the matching + row/column of a `scipy.sparce.csr_matrix`, called `T_small` will be + used to fill the empty cells in the matrix. + + ..Note:: + A concise explanation of how sparse matrices are represented in + `csr`-format can be found + [on StackOverflow](https://stackoverflow.com/a/52299730/1622937). + + + Parameters + ---------- + size: + Determines the number of rows/columns in of the matrix + data: + See `scipy.sparse.csr_matrix` for details + indices: + See `scipy.sparse.csr_matrix` for details + indptr: + See `scipy.sparse.csr_matrix` for details + nz_rowcols: + A collection of column (or row) indexes into which the columns + (or rows) of `T_small` map. + + For all index values < `size`, the corresponding rows and columns + will be filled with the row/column from a diagnoal matrix if the + index is not present in `nz_rowcols` and + + ..Note:: + The number of elements in `nz_rowcols` must match the size + of `T_small`. + diag_val: + The value to use on the diagnoal in diagonal row/colums. + + + """ self.size = size self.nz_rowcols = np.unique(np.array(nz_rowcols, dtype=np.int32)) #sorted unique @@ -118,7 +159,38 @@ def __init__(self, size, data, indices, indptr, nz_rowcols, @classmethod def from_small_csr_matrix(cls, size:int, T_small:csr_matrix, nz_rowcols:NDArray, diag_val:float=1.0): - """Initialize sparse_stoch_mat from a small csr_matrix""" + """Initialize sparse_stoch_mat from a small csr_matrix + + The sparse_stoch_mat will be a square matrix of size `size` with + row/columns of a diagonal matrix for every row/column index not present + in `nz_rowcols`. For the row/column indices present in `nz_rowcols` + the matching row/column of `T_small` will be used to fill the empty + cells in the matrix. + + + Parameters + ---------- + size: + Determines the number of rows/columns in of the matrix + T_small: + A scipy.sparse.csr_matrix that does not contain any rows/columns + of a diagonal matrix. + nz_rowcols: + A collection of column (or row) indexes into which the columns + (or rows) of `T_small` map. + + For all index values < `size`, the corresponding rows and columns + will be filled with the row/column from a diagnoal matrix if the + index is not present in `nz_rowcols` and + + ..Note:: + The number of elements in `nz_rowcols` must match the size + of `T_small`. + diag_val: + The value to use on the diagnoal in diagonal row/colums. + + + """ if not isspmatrix_csr(T_small): raise TypeError("T_small must be in CSR format.") From cb957104568d109256e70da15674d278a24be156 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 21 Aug 2024 12:08:09 +0200 Subject: [PATCH 11/86] using also __init__ docstring --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 05c73c8..4362b67 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,4 +44,5 @@ autoapi_dirs = ["../src/", ] autoapi_file_patterns = ['*.py', ] autoapi_member_order = "groupwise" +autoclass_content = "both" # use docstring of both class and its __init__ # autoapi_ignore = ["*conf.py", "*setup.py" , "*_cython*.pyx", ] From 9bdfb3e510c955396a2c6572560ede9693e7b019 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 21 Aug 2024 12:08:46 +0200 Subject: [PATCH 12/86] minor formating corection --- src/flowstab/SparseStochMat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 2e49043..9fd727e 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -112,7 +112,7 @@ def __init__(self, size, data, indices, indptr, nz_rowcols, row/column of a `scipy.sparce.csr_matrix`, called `T_small` will be used to fill the empty cells in the matrix. - ..Note:: + .. note:: A concise explanation of how sparse matrices are represented in `csr`-format can be found [on StackOverflow](https://stackoverflow.com/a/52299730/1622937). From a44f86cfdd1c1b366aadb7b7c5f1446965f35bfa Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 21 Aug 2024 12:18:59 +0200 Subject: [PATCH 13/86] added autodoc ext to sphinx --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 4362b67..f986263 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,6 +25,7 @@ "myst_parser", "sphinx.ext.napoleon", "sphinx.ext.viewcode", + "sphinx.ext.autodoc", "autoapi.extension", ] From 82e7c787ffa974c3952c949d5497fde35d6f3dc0 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 21 Aug 2024 13:07:00 +0200 Subject: [PATCH 14/86] fixing python version also for rtd --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 053e451..ee875a9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.12" + python: "3.9" # Build documentation in the "docs/" directory with Sphinx sphinx: From e4fc254192f0fafe009f41d85672338859d6aa6b Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 21 Aug 2024 13:07:31 +0200 Subject: [PATCH 15/86] configure autoapi instead of autodoc --- docs/conf.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f986263..414fc42 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,10 +5,8 @@ import pathlib import sys -# -- Add the project root for autodiscovery with sphinx.ext.autodoc ---------- +# -- Add the project root for autodiscovery ---------- sys.path.insert(0, pathlib.Path(__file__).parents[1].resolve().as_posix()) -autodoc_typehints = 'description' -autodoc_class_signature = 'separated' # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information @@ -25,7 +23,6 @@ "myst_parser", "sphinx.ext.napoleon", "sphinx.ext.viewcode", - "sphinx.ext.autodoc", "autoapi.extension", ] @@ -45,5 +42,5 @@ autoapi_dirs = ["../src/", ] autoapi_file_patterns = ['*.py', ] autoapi_member_order = "groupwise" -autoclass_content = "both" # use docstring of both class and its __init__ +autoapi_python_class_content = "both" # autoapi_ignore = ["*conf.py", "*setup.py" , "*_cython*.pyx", ] From 55a12fb8af20ed2300857dbd3dc2559518f4ce05 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Thu, 22 Aug 2024 13:25:17 +0200 Subject: [PATCH 16/86] separate memory tracking tests with _memory in name --- .github/workflows/development.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 0dedc13..26ec35c 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -95,9 +95,12 @@ jobs: with: path: ./venv key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - - name: Test with pytest + - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 --memray + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 -k 'not _memory' + - name: Tests with memory tracking + run: | + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 --memray -k '_memory' lint: runs-on: ubuntu-latest From 570ed07685abd4e0e2d12800c84b252ff0b374dc Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Thu, 22 Aug 2024 13:25:52 +0200 Subject: [PATCH 17/86] better typing --- src/flowstab/SparseStochMat.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 9fd727e..3db2550 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -23,6 +23,7 @@ import importlib.util import os import time +from collections.abc import Callable from copy import copy from functools import wraps @@ -75,7 +76,9 @@ print("Could not load sparse_dot_mkl. Will use scipy.sparse for matrix products.") # timing decorator -def timing(f): +def timing(f:Callable)->Callable: + """ + """ @wraps(f) def wrapper(*args, **kwargs): start = time.time() @@ -101,8 +104,8 @@ class sparse_stoch_mat: """ - def __init__(self, size, data, indices, indptr, nz_rowcols, - diag_val=1.0): + def __init__(self, size:int, data:NDArray, indices:NDArray, indptr:NDArray, + nz_rowcols:NDArray, diag_val:float=1.0): """Initialize sparse_stoch_mat The sparse_stoch_mat will be a square matrix of size `size` with @@ -158,7 +161,7 @@ def __init__(self, size, data, indices, indptr, nz_rowcols, @classmethod def from_small_csr_matrix(cls, size:int, T_small:csr_matrix, nz_rowcols:NDArray, - diag_val:float=1.0): + diag_val:float=1.0)->sparse_stoch_mat: """Initialize sparse_stoch_mat from a small csr_matrix The sparse_stoch_mat will be a square matrix of size `size` with @@ -199,7 +202,7 @@ def from_small_csr_matrix(cls, size:int, T_small:csr_matrix, nz_rowcols:NDArray, @classmethod def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, - diag_val:float=1.0): + diag_val:float=1.0)->sparse_stoch_mat: """Initialize sparse_stoch_mat from a full size row stochastic csr_matrix """ @@ -251,7 +254,7 @@ def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, nz_rowcols, diag_val=diag_val) @classmethod - def create_diag(cls, size, diag_val=1.0): + def create_diag(cls, size:int, diag_val:float=1.0)->sparse_stoch_mat: """Returns a diagonal matrix with an empty T_small. Parameters @@ -266,7 +269,7 @@ def create_diag(cls, size, diag_val=1.0): return cls.from_small_csr_matrix(size, T_small, [], diag_val=diag_val) - def inplace_row_normalize(self, row_sum=1.0): + def inplace_row_normalize(self, row_sum:float=1.0): if USE_CYTHON: self.T_small.indptr = self.T_small.indptr.astype(np.int64, copy=False) @@ -283,7 +286,7 @@ def inplace_row_normalize(self, row_sum=1.0): self.diag_val = row_sum - def set_to_zeroes(self, tol=1e-8, relative=True, use_absolute_value=False): + def set_to_zeroes(self, tol:float=1e-8, relative:bool=True, use_absolute_value:bool=False): """In place replaces zeroes in the T_small sparse matrix that are, within the tolerence, close to zero with actual zeroes """ @@ -299,7 +302,7 @@ def set_to_zeroes(self, tol=1e-8, relative=True, use_absolute_value=False): self.T_small.eliminate_zeros() - def to_full_mat(self): + def to_full_mat(self)->csr_matrix: """Returns a full size sparse matrix""" return rebuild_nnz_rowcol(self.T_small, self.nz_rowcols, @@ -1133,7 +1136,8 @@ def inplace_csr_row_normalize(X, row_sum=1.0): else: raise TypeError("X must be in ndarray, CSR or sparse_stoch_mat format.") -def rebuild_nnz_rowcol(T_small, nonzero_indices, size, diag_val=1.0): +def rebuild_nnz_rowcol(T_small:csr_matrix, nonzero_indices:NDArray, + size:int, diag_val:float=1.0)->csr_matrix: """Returns a CSR matrix built from the CSR matrix T_small with T_small values at row-colums corresponding to nonzero_indices and 1 on the diagonal elsewhere. From 2f8d2327637d178091152d66b90505dd77a80ec9 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Thu, 22 Aug 2024 15:33:45 +0200 Subject: [PATCH 18/86] make cython and non-cython implementatios testable --- src/flowstab/SparseStochMat.py | 47 ++++-------------- src/flowstab/_cython_sparse_stoch.pyx | 2 +- src/flowstab/_cython_sparse_stoch_subst.py | 30 ++++++++++++ tests/test_sparse_stoch_matrix.py | 55 +++++++++++++++++++++- 4 files changed, 94 insertions(+), 40 deletions(-) create mode 100644 src/flowstab/_cython_sparse_stoch_subst.py diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 3db2550..f43bd67 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -41,6 +41,7 @@ USE_CYTHON = True if importlib.util.find_spec("cython") is not None: + import _cython_sparse_stoch as _css from _cython_sparse_stoch import ( cython_aggregate_csr_mat, cython_aggregate_csr_mat_2, @@ -56,13 +57,14 @@ cython_inplace_csr_row_normalize, cython_inplace_csr_row_normalize_array, cython_rebuild_nnz_rowcol, - cython_sparse_stoch_from_full_csr, + sparse_stoch_from_full_csr as cython_sparse_stoch_from_full_csr, cython_stoch_mat_add, cython_stoch_mat_sub, ) else: print("Could not load cython functions. Some functionality might be broken.") + from . import _cython_sparse_stoch_subst as _css USE_CYTHON = False @@ -216,42 +218,13 @@ def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, # Make sure we work with floats Tcsr_data = Tcsr.data.astype(np.float64) - if USE_CYTHON: - - res = cython_sparse_stoch_from_full_csr( - np.array(nz_rowcols, dtype=np.int32), - Tcsr_data, - Tcsr.indices, - Tcsr.indptr, - diag_val - ) - - return cls(*res) - - else: - - T_s_nnz = Tcsr.nnz - Tcsr.shape[0] + nz_rowcols.size - - T_s_data = np.zeros(T_s_nnz, dtype=np.float64) - T_s_indices = -1*np.ones(T_s_nnz, dtype=np.int32) - T_s_indptr = -1*np.ones(nz_rowcols.size+1, dtype=np.int32) - - #map indices from big to small T - BtoS = {v:k for k,v in enumerate(nz_rowcols)} - - its = 0 - T_s_indptr[0] = 0 - for tsrow, tbrow in enumerate(nz_rowcols): - nzr = 0 - for k in range(Tcsr.indptr[tbrow],Tcsr.indptr[tbrow+1]): - T_s_data[its] = Tcsr_data[k] - T_s_indices[its] = BtoS[Tcsr.indices[k]] - its += 1 - nzr += 1 - T_s_indptr[tsrow+1] = T_s_indptr[tsrow] + nzr - - return cls(Tcsr.shape[0], T_s_data, T_s_indices, T_s_indptr, - nz_rowcols, diag_val=diag_val) + return cls(*_css.sparse_stoch_from_full_csr( + np.array(nz_rowcols, dtype=np.int32), + Tcsr_data, + Tcsr.indices, + Tcsr.indptr, + diag_val + )) @classmethod def create_diag(cls, size:int, diag_val:float=1.0)->sparse_stoch_mat: diff --git a/src/flowstab/_cython_sparse_stoch.pyx b/src/flowstab/_cython_sparse_stoch.pyx index 02f7cb8..ccbfa29 100644 --- a/src/flowstab/_cython_sparse_stoch.pyx +++ b/src/flowstab/_cython_sparse_stoch.pyx @@ -856,7 +856,7 @@ def cython_inplace_csr_row_normalize_triu(double[:] X_data, @cython.boundscheck(False) # Deactivate bounds checking @cython.wraparound(False) # Deactivate negative indexing -def cython_sparse_stoch_from_full_csr(int[:] nz_rowcols, +def sparse_stoch_from_full_csr(int[:] nz_rowcols, double[:] Tf_data, int[:] Tf_indices, int[:] Tf_indptr, diff --git a/src/flowstab/_cython_sparse_stoch_subst.py b/src/flowstab/_cython_sparse_stoch_subst.py new file mode 100644 index 0000000..546df84 --- /dev/null +++ b/src/flowstab/_cython_sparse_stoch_subst.py @@ -0,0 +1,30 @@ +import numpy as np + +def sparse_stoch_from_full_csr(nz_rowcols, Tf_data, Tf_indices, Tf_indptr, diag_val): + """Pure python implementation of the sparce_stoch_mat + """ + T_s_nnz = Tf_data.shape[0] - Tf_indptr.shape[0] + 1 + nz_rowcols.shape[0] + + T_s_data = np.zeros(T_s_nnz, dtype=np.float64) + T_s_indices = -1*np.ones(T_s_nnz, dtype=np.int32) + T_s_indptr = -1*np.ones(nz_rowcols.size+1, dtype=np.int32) + + #map indices from big to small T + BtoS = {v:k for k,v in enumerate(nz_rowcols)} + + its = 0 + T_s_indptr[0] = 0 + for tsrow, tbrow in enumerate(nz_rowcols): + nzr = 0 + for k in range(Tf_indptr[tbrow],Tf_indptr[tbrow+1]): + T_s_data[its] = Tf_data[k] + T_s_indices[its] = BtoS[Tf_indices[k]] + its += 1 + nzr += 1 + T_s_indptr[tsrow+1] = T_s_indptr[tsrow] + nzr + return (Tf_indptr.shape[0] - 1, + T_s_data, + T_s_indices, + T_s_indptr, + nz_rowcols, + diag_val) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 78fed4d..906c352 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -1,6 +1,10 @@ import numpy as np import tracemalloc +from scipy.sparse import ( + eye, +) + def test_SSM_small(get_csr_matrix_small): """Bacic operations with the 'spares_stoch_mat' class """ @@ -23,16 +27,63 @@ def test_SSM_large(get_csr_matrix_large): # inti from scipy.sparse.csr_matrix A_csr, density = get_csr_matrix_large tracemalloc.start() - for i in range(100): + for _ in range(100): _ = SSM.from_full_csr_matrix(A_csr) first_size, first_peak = tracemalloc.get_traced_memory() - for i in range(100): + tracemalloc.reset_peak() + + for _ in range(100): _ = SSM.from_full_csr_matrix(A_csr).toarray() second_size, second_peak = tracemalloc.get_traced_memory() assert first_size < density * second_size assert first_peak < density * second_peak +def test_SSM_from_full_csr_cython_memory(get_csr_matrix_large): + """Check the cython implementation + """ + from flowstab.SparseStochMat import ( + _css + ) + A_csr, density = get_csr_matrix_large + A_csr_data = A_csr.data.astype(np.float64) + diag_val = 1.0 + nz_rows, nz_cols = ( + A_csr - diag_val * eye(A_csr.shape[0], format="csr") + ).nonzero() + nz_rowcols = np.union1d(nz_rows, nz_cols) + for _ in range(100): + _ = _css.sparse_stoch_from_full_csr( + np.array(nz_rowcols, dtype=np.int32), + A_csr_data, + A_csr.indices, + A_csr.indptr, + diag_val + ) + + +def test_SSM_from_full_csr_nocython_memory(get_csr_matrix_large): + """Check the python substitue + """ + from flowstab._cython_sparse_stoch_subst import ( + sparse_stoch_from_full_csr as sparse_stoch_from_full_csr + ) + A_csr, density = get_csr_matrix_large + A_csr_data = A_csr.data.astype(np.float64) + diag_val = 1.0 + nz_rows, nz_cols = ( + A_csr - diag_val * eye(A_csr.shape[0], format="csr") + ).nonzero() + nz_rowcols = np.union1d(nz_rows, nz_cols) + for _ in range(100): + _ = sparse_stoch_from_full_csr( + np.array(nz_rowcols, dtype=np.int32), + A_csr_data, + A_csr.indices, + A_csr.indptr, + diag_val + ) + def test_SPA(): """Basic operations with the `SpasreStochMat.SPA` class""" From e4f5db5b64e8326e085bbaf0906b08d385734744 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Thu, 22 Aug 2024 18:49:53 +0200 Subject: [PATCH 19/86] separating inplace row norm --- src/flowstab/SparseStochMat.py | 28 ++++----- src/flowstab/_cython_sparse_stoch.pyx | 16 ++--- src/flowstab/_cython_sparse_stoch_subst.py | 10 ++++ tests/conftest.py | 15 +++++ tests/test_sparse_stoch_matrix.py | 68 ++++++++++++++++++++++ 5 files changed, 112 insertions(+), 25 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index f43bd67..a604552 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -42,6 +42,7 @@ USE_CYTHON = True if importlib.util.find_spec("cython") is not None: import _cython_sparse_stoch as _css + from _cython_sparse_stoch import ( cython_aggregate_csr_mat, cython_aggregate_csr_mat_2, @@ -54,20 +55,18 @@ cython_csr_csrT_matmul, cython_csr_matmul, cython_get_submat_sum, - cython_inplace_csr_row_normalize, + inplace_csr_row_normalize as cython_inplace_csr_row_normalize, cython_inplace_csr_row_normalize_array, cython_rebuild_nnz_rowcol, sparse_stoch_from_full_csr as cython_sparse_stoch_from_full_csr, cython_stoch_mat_add, cython_stoch_mat_sub, ) - else: print("Could not load cython functions. Some functionality might be broken.") from . import _cython_sparse_stoch_subst as _css USE_CYTHON = False - USE_SPARSE_DOT_MKL = True if importlib.util.find_spec("sparse_dot_mkl") is not None: from sparse_dot_mkl import dot_product_mkl, gram_matrix_mkl @@ -205,8 +204,7 @@ def from_small_csr_matrix(cls, size:int, T_small:csr_matrix, nz_rowcols:NDArray, @classmethod def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, diag_val:float=1.0)->sparse_stoch_mat: - """Initialize sparse_stoch_mat from a full size row stochastic - csr_matrix + """Init sparse_stoch_mat from a full size row stochastic csr_matrix """ if not isspmatrix_csr(Tcsr): raise TypeError("T_small must be in CSR format.") @@ -240,25 +238,21 @@ def create_diag(cls, size:int, diag_val:float=1.0)->sparse_stoch_mat: """ T_small = csr_matrix((0,0)) - return cls.from_small_csr_matrix(size, T_small, [], diag_val=diag_val) + return cls.from_small_csr_matrix(size, T_small, np.array([]), diag_val=diag_val) def inplace_row_normalize(self, row_sum:float=1.0): + """Normalize the rows in place + """ - if USE_CYTHON: - self.T_small.indptr = self.T_small.indptr.astype(np.int64, copy=False) - self.T_small.indices = self.T_small.indices.astype(np.int64, copy=False) - - cython_inplace_csr_row_normalize(self.T_small.data, self.T_small.indptr, - self.T_small.shape[0], row_sum) - else: + self.T_small.indptr = self.T_small.indptr.astype(np.int64, copy=False) + self.T_small.indices = self.T_small.indices.astype(np.int64, copy=False) - for i in range(self.T_small.shape[0]): - row_sum_tmp = self.T_small.data[self.T_small.indptr[i]:self.T_small.indptr[i+1]].sum() - if row_sum_tmp != 0: - self.T_small.data[self.T_small.indptr[i]:self.T_small.indptr[i+1]] /= (row_sum_tmp/row_sum) + _css.inplace_csr_row_normalize(self.T_small.data, self.T_small.indptr, + self.T_small.shape[0], row_sum) self.diag_val = row_sum + def set_to_zeroes(self, tol:float=1e-8, relative:bool=True, use_absolute_value:bool=False): """In place replaces zeroes in the T_small sparse matrix that are, within the tolerence, close to zero with actual zeroes diff --git a/src/flowstab/_cython_sparse_stoch.pyx b/src/flowstab/_cython_sparse_stoch.pyx index ccbfa29..d82ab2a 100644 --- a/src/flowstab/_cython_sparse_stoch.pyx +++ b/src/flowstab/_cython_sparse_stoch.pyx @@ -698,10 +698,10 @@ def cython_rebuild_nnz_rowcol(double[:] T_data, @cython.boundscheck(False) # Deactivate bounds checking @cython.wraparound(False) # Deactivate negative indexing @cython.cdivision(True) -def cython_inplace_csr_row_normalize(double[:] X_data, - long long[:] X_indptr, - Py_ssize_t n_row, - double row_sum=1.0): +def inplace_csr_row_normalize(double[:] X_data, + long long[:] X_indptr, + Py_ssize_t n_row, + double row_sum=1.0): """ row normalize scipy sparse csr matrices inplace. inspired from sklearn sparsefuncs_fast.pyx. @@ -857,10 +857,10 @@ def cython_inplace_csr_row_normalize_triu(double[:] X_data, @cython.boundscheck(False) # Deactivate bounds checking @cython.wraparound(False) # Deactivate negative indexing def sparse_stoch_from_full_csr(int[:] nz_rowcols, - double[:] Tf_data, - int[:] Tf_indices, - int[:] Tf_indptr, - double diag_val): + double[:] Tf_data, + long long[:] Tf_indices, + int[:] Tf_indptr, + double diag_val): """ initialize sparse_stoch_mat from a full size row stochastic csr_matrix diff --git a/src/flowstab/_cython_sparse_stoch_subst.py b/src/flowstab/_cython_sparse_stoch_subst.py index 546df84..624f887 100644 --- a/src/flowstab/_cython_sparse_stoch_subst.py +++ b/src/flowstab/_cython_sparse_stoch_subst.py @@ -28,3 +28,13 @@ def sparse_stoch_from_full_csr(nz_rowcols, Tf_data, Tf_indices, Tf_indptr, diag_ T_s_indptr, nz_rowcols, diag_val) + + +def inplace_csr_row_normalize(X_data, + X_indptr, + n_row, # T_small.shape[0], + row_sum): + for i in range(n_row): + row_sum_tmp = X_data[X_indptr[i]:X_indptr[i+1]].sum() + if row_sum_tmp != 0: + X_data[X_indptr[i]:X_indptr[i+1]] = (row_sum_tmp/row_sum) diff --git a/tests/conftest.py b/tests/conftest.py index 80561d3..aef98c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,3 +23,18 @@ def get_csr_matrix_large(): data = np.random.randint(0,100, size=nbr_non_zeros) density = nbr_non_zeros / size return csr_matrix((data, (row, col)), shape=(size, size)), density + +@pytest.fixture(scope='function') +def get_SSM_matrix_large(): + """Creat an exemplary csr matrix that can be used for testing + """ + from flowstab.SparseStochMat import sparse_stoch_mat + size = 1000000 + nbr_non_zeros = 1000 + row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + data = np.random.randint(0,1, size=nbr_non_zeros) + _a_csr = csr_matrix((data, (row, col)), shape=(size, size)) + _a_csr.indptr = _a_csr.indptr.astype(np.int64, copy=False) + _a_csr.indices = _a_csr.indices.astype(np.int64, copy=False) + return sparse_stoch_mat.from_full_csr_matrix(_a_csr) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 906c352..4d4f5d0 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -1,6 +1,8 @@ import numpy as np import tracemalloc +from copy import copy + from scipy.sparse import ( eye, ) @@ -84,6 +86,72 @@ def test_SSM_from_full_csr_nocython_memory(get_csr_matrix_large): diag_val ) +def test_SSM_from_full_csr_equivalence(get_csr_matrix_large): + """Check if both cython and native python implementations match + """ + from flowstab.SparseStochMat import ( + _css + ) + from flowstab._cython_sparse_stoch_subst import ( + sparse_stoch_from_full_csr as sparse_stoch_from_full_csr + ) + A_csr, _ = get_csr_matrix_large + A_csr_data = A_csr.data.astype(np.float64) + diag_val = 1.0 + nz_rows, nz_cols = ( + A_csr - diag_val * eye(A_csr.shape[0], format="csr") + ).nonzero() + nz_rowcols = np.union1d(nz_rows, nz_cols) + ( + c_size, c_data, c_indices, + c_indptr, c_nz_rowcols, c_diag_val + ) = _css.sparse_stoch_from_full_csr( + np.array(nz_rowcols, dtype=np.int32), + A_csr_data, + A_csr.indices, + A_csr.indptr, + diag_val + ) + ( + nc_size, nc_data, nc_indices, + nc_indptr, nc_nz_rowcols, nc_diag_val + ) = sparse_stoch_from_full_csr( + np.array(nz_rowcols, dtype=np.int32), + A_csr_data, + A_csr.indices, + A_csr.indptr, + diag_val + ) + assert nc_size == c_size + assert nc_diag_val == c_diag_val + np.testing.assert_array_equal(nc_data, c_data) + np.testing.assert_array_equal(nc_indices, c_indices) + np.testing.assert_array_equal(nc_indptr, c_indptr) + np.testing.assert_array_equal(nc_nz_rowcols, c_nz_rowcols) + +def test_SSM_inplace_row_normalize_equivalence(get_SSM_matrix_large): + """Make sure the cython and pure python implementations are equivalent + """ + from flowstab.SparseStochMat import ( + _css + ) + from flowstab._cython_sparse_stoch_subst import ( + inplace_csr_row_normalize + ) + A_ssm1 = get_SSM_matrix_large + A_ssm1_data = copy(A_ssm1.T_small.data) + A_ssm2 = copy(A_ssm1) + A_ssm2_data = copy(A_ssm2.T_small.data) + # the cython implementation + _css.inplace_csr_row_normalize(A_ssm1.T_small.data, A_ssm1.T_small.indptr, A_ssm1.T_small.shape[0]) + # pure python + inplace_csr_row_normalize(A_ssm2.T_small.data, A_ssm2.T_small.indptr, A_ssm2.T_small.shape[0]) + # test change + np.testing.assert_array_equal(A_ssm1_data, A_ssm1.T_small.data) + # test equivalence + np.testing.assert_array_equal(A_ssm1.data, A_ssm2.T_small.data) + + def test_SPA(): """Basic operations with the `SpasreStochMat.SPA` class""" From ce843af9b900634c1e13e7922fb8621f5351ac6d Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Thu, 22 Aug 2024 19:45:00 +0200 Subject: [PATCH 20/86] try to satisfy long long vs int --- tests/test_sparse_stoch_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 4d4f5d0..71caa89 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -108,7 +108,7 @@ def test_SSM_from_full_csr_equivalence(get_csr_matrix_large): ) = _css.sparse_stoch_from_full_csr( np.array(nz_rowcols, dtype=np.int32), A_csr_data, - A_csr.indices, + A_csr.indices.astype(np.int64), A_csr.indptr, diag_val ) From bb750ec7b0b5d20fdfa966b819135b42173f83c0 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 13:21:43 +0200 Subject: [PATCH 21/86] minor - avoid declaring unused params --- tests/test_sparse_stoch_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 71caa89..67bebbf 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -15,7 +15,7 @@ def test_SSM_small(get_csr_matrix_small): # ### # inti from scipy.sparse.csr_matrix A_csr = get_csr_matrix_small - for i in range(10000): + for _ in range(10000): ssm = SSM.from_full_csr_matrix(A_csr) np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) @@ -47,7 +47,7 @@ def test_SSM_from_full_csr_cython_memory(get_csr_matrix_large): from flowstab.SparseStochMat import ( _css ) - A_csr, density = get_csr_matrix_large + A_csr, _ = get_csr_matrix_large A_csr_data = A_csr.data.astype(np.float64) diag_val = 1.0 nz_rows, nz_cols = ( From eb27b743b0c0c9ef0b76a9ae5bbc54a3d38539d5 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 13:53:37 +0200 Subject: [PATCH 22/86] target src for coverage --- .github/workflows/development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 26ec35c..f3c4acd 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 -k 'not _memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --cov-report=xml --durations=0 -k 'not _memory' - name: Tests with memory tracking run: | venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 --memray -k '_memory' From 005cf7ca5eaa46c9124325b0cdda0bfb4d5eccbe Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 13:54:45 +0200 Subject: [PATCH 23/86] config for test coverage --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9df7172..d1e27d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,15 @@ requires = [ ] build-backend = "setuptools.build_meta" +# config for the test coverage +[tool.coverage.run] +plugins = ["Cython.Coverage", ] +source = ["src/flowstab/", ] +omit = [ + "tests", + "*__init__.py", + ] + # This is for the linting with ruff [tool.ruff] target-version = "py39" From 49bf5052e3ef7e1257de079269a15863ce35f17f Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 13:55:25 +0200 Subject: [PATCH 24/86] cleaning up --- tests/test_sparse_stoch_matrix.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 67bebbf..4aa5de6 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -143,17 +143,11 @@ def test_SSM_inplace_row_normalize_equivalence(get_SSM_matrix_large): A_ssm2 = copy(A_ssm1) A_ssm2_data = copy(A_ssm2.T_small.data) # the cython implementation - _css.inplace_csr_row_normalize(A_ssm1.T_small.data, A_ssm1.T_small.indptr, A_ssm1.T_small.shape[0]) + _css.inplace_csr_row_normalize(A_ssm1.T_small.data, A_ssm1.T_small.indptr, A_ssm1.T_small.shape[0], 1.0) # pure python - inplace_csr_row_normalize(A_ssm2.T_small.data, A_ssm2.T_small.indptr, A_ssm2.T_small.shape[0]) + inplace_csr_row_normalize(A_ssm2.T_small.data, A_ssm2.T_small.indptr, A_ssm2.T_small.shape[0], 1.0) # test change np.testing.assert_array_equal(A_ssm1_data, A_ssm1.T_small.data) + np.testing.assert_array_equal(A_ssm2_data, A_ssm2.T_small.data) # test equivalence np.testing.assert_array_equal(A_ssm1.data, A_ssm2.T_small.data) - - - -def test_SPA(): - """Basic operations with the `SpasreStochMat.SPA` class""" - from flowstab.SparseStochMat import SPA - # TODO From 75021ed282b51cb5b5e15926687098b5586ef5f7 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 13:56:08 +0200 Subject: [PATCH 25/86] allow coverage to trace lines --- src/flowstab/_cython_sparse_stoch.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/flowstab/_cython_sparse_stoch.pyx b/src/flowstab/_cython_sparse_stoch.pyx index d82ab2a..c8a6a1c 100644 --- a/src/flowstab/_cython_sparse_stoch.pyx +++ b/src/flowstab/_cython_sparse_stoch.pyx @@ -1,5 +1,7 @@ # distutils: language = c++ # cython: profile=False +# cython: linetrace=True + """ # # flow stability From 03b1c5a73c068c3da27bd684fd4b03e24994f27b Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:03:36 +0200 Subject: [PATCH 26/86] reporting code coverage --- .github/workflows/development.yml | 45 +++++++++++++++++++++++++++ .github/workflows/report_coverage.yml | 29 +++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 .github/workflows/report_coverage.yml diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index f3c4acd..7d174d9 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -98,9 +98,54 @@ jobs: - name: Tests without memory tracking run: | venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --cov-report=xml --durations=0 -k 'not _memory' + env: + COVERAGE_FILE: ".coverage.no_memory" + - name: Store coverage file + uses: actions/upload-artifact@v4 + with: + name: coverage-no_memory + path: .coverage.no_memory - name: Tests with memory tracking run: | venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 --memray -k '_memory' + env: + COVERAGE_FILE: ".coverage.memory" + - name: Store coverage file + uses: actions/upload-artifact@v4 + with: + name: coverage-memory + path: .coverage.memory + + coverage: + name: Combine coverage + runs-on: ubuntu-22.04 + needs: build + permissions: + pull-requests: write + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + id: download + with: + pattern: coverage-* + merge-multiple: true + + - name: Coverage comment + id: coverage_comment + uses: py-cov-action/python-coverage-comment-action@v3 + with: + GITHUB_TOKEN: ${{ github.token }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MERGE_COVERAGE_FILES: true + + - name: Store Pull Request comment to be posted + uses: actions/upload-artifact@v4 + if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true' + with: + name: python-coverage-comment-action + path: python-coverage-comment-action.txt lint: runs-on: ubuntu-latest diff --git a/.github/workflows/report_coverage.yml b/.github/workflows/report_coverage.yml new file mode 100644 index 0000000..5c8b56e --- /dev/null +++ b/.github/workflows/report_coverage.yml @@ -0,0 +1,29 @@ +name: Post coverage comment + +on: + workflow_run: + workflows: ["Development Workflow"] + types: + - completed + +jobs: + test: + name: Run tests & display coverage + runs-on: ubuntu-22.04 + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + permissions: + pull-requests: write + contents: write + actions: read + steps: + # DO NOT run actions/checkout here, for security reasons + # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + - name: Post comment + uses: py-cov-action/python-coverage-comment-action@v3 + with: + GITHUB_TOKEN: ${{ github.token }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }} + # Update those if you changed the default values: + # COMMENT_ARTIFACT_NAME: python-coverage-comment-action + # COMMENT_FILENAME: python-coverage-comment-action.txt From 46b986fb37490a153d35ace8cccdd61236db6116 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:05:23 +0200 Subject: [PATCH 27/86] establish jobs order --- tests/test_sparse_stoch_matrix.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 4aa5de6..ba34530 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -7,6 +7,21 @@ eye, ) +def test_timing(capfd): + """Bacic operations with the 'spares_stoch_mat' class + """ + from time import sleep + from flowstab.SparseStochMat import timing + + @timing + def sleep_some(some=0.3): + return sleep(some) + sleep_some() + out, err = capfd.readouterr() + print(out) + print(err) + + def test_SSM_small(get_csr_matrix_small): """Bacic operations with the 'spares_stoch_mat' class """ From 60b1e2c73db8a7060b4e955ef80ec0e617841510 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:07:05 +0200 Subject: [PATCH 28/86] using correct secret --- .github/workflows/development.yml | 7 +++---- .github/workflows/report_coverage.yml | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 7d174d9..576fbfc 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -117,9 +117,9 @@ jobs: path: .coverage.memory coverage: - name: Combine coverage + name: combine-coverage runs-on: ubuntu-22.04 - needs: build + needs: test permissions: pull-requests: write contents: write @@ -136,8 +136,7 @@ jobs: id: coverage_comment uses: py-cov-action/python-coverage-comment-action@v3 with: - GITHUB_TOKEN: ${{ github.token }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MERGE_COVERAGE_FILES: true - name: Store Pull Request comment to be posted diff --git a/.github/workflows/report_coverage.yml b/.github/workflows/report_coverage.yml index 5c8b56e..32e0e5c 100644 --- a/.github/workflows/report_coverage.yml +++ b/.github/workflows/report_coverage.yml @@ -21,8 +21,7 @@ jobs: - name: Post comment uses: py-cov-action/python-coverage-comment-action@v3 with: - GITHUB_TOKEN: ${{ github.token }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }} # Update those if you changed the default values: # COMMENT_ARTIFACT_NAME: python-coverage-comment-action From c785c4a71c7c91347666c63efb4eb63aeefd6704 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:12:50 +0200 Subject: [PATCH 29/86] always run coverage --- .github/workflows/development.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 576fbfc..6bcafd2 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -120,6 +120,8 @@ jobs: name: combine-coverage runs-on: ubuntu-22.04 needs: test + if: | + always() permissions: pull-requests: write contents: write From 287748439ef2c52a64006670345304f8cdc8e4cf Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:14:22 +0200 Subject: [PATCH 30/86] run coverage report --- .github/workflows/report_coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/report_coverage.yml b/.github/workflows/report_coverage.yml index 32e0e5c..6b571e3 100644 --- a/.github/workflows/report_coverage.yml +++ b/.github/workflows/report_coverage.yml @@ -10,7 +10,7 @@ jobs: test: name: Run tests & display coverage runs-on: ubuntu-22.04 - if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + if: github.event.workflow_run.event == 'pull_request' permissions: pull-requests: write contents: write From 2e6b87c792fd3ce05fc50ed6f23c17584b2228c0 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:25:16 +0200 Subject: [PATCH 31/86] remove plugin for codecov reports --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d1e27d5..3239f30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,8 @@ build-backend = "setuptools.build_meta" # config for the test coverage [tool.coverage.run] -plugins = ["Cython.Coverage", ] +relative_files = true +# plugins = ["Cython.Coverage", ] source = ["src/flowstab/", ] omit = [ "tests", From 73909794bb03ad32d397b3fefe61b1721d3b9f35 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 15:35:17 +0200 Subject: [PATCH 32/86] single folder several files --- .github/workflows/development.yml | 33 ++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 6bcafd2..7d03bd3 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,29 +97,48 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --cov-report=xml --durations=0 -k 'not _memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --durations=0 -k 'not _memory' env: - COVERAGE_FILE: ".coverage.no_memory" + COVERAGE_FILE: "_coverage.no_memory" + - name: Store coverage file + if: always() uses: actions/upload-artifact@v4 with: name: coverage-no_memory - path: .coverage.no_memory + path: _coverage.no_memory + + test-memory: + runs-on: ubuntu-latest + needs: install + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: 3.9 + # Note: we use cache here since artifacts do not keep permissions + - name: Getting the venv + uses: actions/cache/restore@v4 + with: + path: ./venv + key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./ --cov-report=xml --durations=0 --memray -k '_memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --durations=0 --memray -k '_memory' env: - COVERAGE_FILE: ".coverage.memory" + COVERAGE_FILE: "_coverage.memory" - name: Store coverage file + if: always() uses: actions/upload-artifact@v4 with: name: coverage-memory - path: .coverage.memory + path: _coverage.memory coverage: name: combine-coverage runs-on: ubuntu-22.04 - needs: test + needs: [test, test-memory] if: | always() permissions: From a376ede76f93be9fa92229703775e149b40e7fe8 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 16:57:38 +0200 Subject: [PATCH 33/86] specify package location --- .github/workflows/development.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 7d03bd3..ba1246f 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,10 +97,9 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --durations=0 -k 'not _memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src --durations=0 -k 'not _memory' env: COVERAGE_FILE: "_coverage.no_memory" - - name: Store coverage file if: always() uses: actions/upload-artifact@v4 @@ -125,7 +124,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src/flowstab/ --durations=0 --memray -k '_memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src --durations=0 --memray -k '_memory' env: COVERAGE_FILE: "_coverage.memory" - name: Store coverage file From 9dabe70a1515126b1d9bf66159837a23af63744b Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 17:06:42 +0200 Subject: [PATCH 34/86] using local install --- .github/workflows/development.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index ba1246f..fccd8e6 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -72,7 +72,10 @@ jobs: path: dist - name: Install package run: | - venv/bin/python -m pip install dist/*.whl + # venv/bin/python -m pip install dist/*.whl + # NOTE: above code installs the built wheel into the site-packages folder. + # For code coverage, however, we want to use a local install + venv/bin/python -m pip install -e . - name: Cache venv uses: actions/cache/save@v4 with: From bd1cd7cec10d2b6165b12379d3d389c36ce18d75 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 17:12:01 +0200 Subject: [PATCH 35/86] workaround for hidden files --- .github/workflows/development.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index fccd8e6..2b08bb0 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -102,13 +102,14 @@ jobs: run: | venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src --durations=0 -k 'not _memory' env: - COVERAGE_FILE: "_coverage.no_memory" + COVERAGE_FILE: ".coverage.no_memory" - name: Store coverage file if: always() uses: actions/upload-artifact@v4 with: name: coverage-no_memory - path: _coverage.no_memory + path: .coverage.no_memory + include_hidden_files: true test-memory: runs-on: ubuntu-latest @@ -129,13 +130,14 @@ jobs: run: | venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src --durations=0 --memray -k '_memory' env: - COVERAGE_FILE: "_coverage.memory" + COVERAGE_FILE: ".coverage.memory" - name: Store coverage file if: always() uses: actions/upload-artifact@v4 with: name: coverage-memory - path: _coverage.memory + path: .coverage.memory + include_hidden_files: true coverage: name: combine-coverage From 525d809c0254ad39735075196fc1a2c3f0b0a00e Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 17:16:08 +0200 Subject: [PATCH 36/86] site-package install but discover package name --- .github/workflows/development.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 2b08bb0..bb5ea35 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -72,10 +72,7 @@ jobs: path: dist - name: Install package run: | - # venv/bin/python -m pip install dist/*.whl - # NOTE: above code installs the built wheel into the site-packages folder. - # For code coverage, however, we want to use a local install - venv/bin/python -m pip install -e . + venv/bin/python -m pip install dist/*.whl - name: Cache venv uses: actions/cache/save@v4 with: @@ -100,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src --durations=0 -k 'not _memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 -k 'not _memory' env: COVERAGE_FILE: ".coverage.no_memory" - name: Store coverage file @@ -128,7 +125,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=./src --durations=0 --memray -k '_memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray -k '_memory' env: COVERAGE_FILE: ".coverage.memory" - name: Store coverage file From 71f01742a75403b0a855675abda2e327d05b39ba Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 17:22:28 +0200 Subject: [PATCH 37/86] fixign artifact option --- .github/workflows/development.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index bb5ea35..280e4c1 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -106,7 +106,7 @@ jobs: with: name: coverage-no_memory path: .coverage.no_memory - include_hidden_files: true + include-hidden-files: true test-memory: runs-on: ubuntu-latest @@ -134,7 +134,7 @@ jobs: with: name: coverage-memory path: .coverage.memory - include_hidden_files: true + include-hidden-files: true coverage: name: combine-coverage From bf651f16aa73e85be290ed01c261312fbdb90f33 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 17:33:16 +0200 Subject: [PATCH 38/86] sepcify source in config to use relative files --- .github/workflows/development.yml | 6 ++---- .github/workflows/report_coverage.yml | 1 + pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 280e4c1..67d1ad8 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 -k 'not _memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --durations=0 -k 'not _memory' env: COVERAGE_FILE: ".coverage.no_memory" - name: Store coverage file @@ -125,7 +125,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray -k '_memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --durations=0 --memray -k '_memory' env: COVERAGE_FILE: ".coverage.memory" - name: Store coverage file @@ -147,13 +147,11 @@ jobs: contents: write steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 id: download with: pattern: coverage-* merge-multiple: true - - name: Coverage comment id: coverage_comment uses: py-cov-action/python-coverage-comment-action@v3 diff --git a/.github/workflows/report_coverage.yml b/.github/workflows/report_coverage.yml index 6b571e3..b6746ec 100644 --- a/.github/workflows/report_coverage.yml +++ b/.github/workflows/report_coverage.yml @@ -10,6 +10,7 @@ jobs: test: name: Run tests & display coverage runs-on: ubuntu-22.04 + # if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' if: github.event.workflow_run.event == 'pull_request' permissions: pull-requests: write diff --git a/pyproject.toml b/pyproject.toml index 3239f30..4832412 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ build-backend = "setuptools.build_meta" [tool.coverage.run] relative_files = true # plugins = ["Cython.Coverage", ] -source = ["src/flowstab/", ] +source = ["flowstab", ] omit = [ "tests", "*__init__.py", From 27960be9794454fc59de0d28a5e3fd335fa82955 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 17:43:42 +0200 Subject: [PATCH 39/86] fixing coverage options --- .github/workflows/development.yml | 4 ++-- pyproject.toml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 67d1ad8..6ae226d 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -97,7 +97,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests without memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --durations=0 -k 'not _memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 -k 'not _memory' env: COVERAGE_FILE: ".coverage.no_memory" - name: Store coverage file @@ -125,7 +125,7 @@ jobs: key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --durations=0 --memray -k '_memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray -k '_memory' env: COVERAGE_FILE: ".coverage.memory" - name: Store coverage file diff --git a/pyproject.toml b/pyproject.toml index 4832412..b604ca7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,11 +51,16 @@ build-backend = "setuptools.build_meta" [tool.coverage.run] relative_files = true # plugins = ["Cython.Coverage", ] -source = ["flowstab", ] +source = ["src/flowstab", ] omit = [ "tests", "*__init__.py", ] +[tool.coverage.paths] +source = [ + "src/flowstab/", + "venv/lib/python*/site-packages/flowstab/" + ] # This is for the linting with ruff [tool.ruff] From 19a62d452081b6694015370f6ec1b785571ff844 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 18:24:09 +0200 Subject: [PATCH 40/86] minor edits --- tests/test_sparse_stoch_matrix.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index ba34530..5a9c799 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -14,12 +14,13 @@ def test_timing(capfd): from flowstab.SparseStochMat import timing @timing - def sleep_some(some=0.3): + def sleep_some(some=0.3, **params): return sleep(some) - sleep_some() + log_message = "END" + sleep_some(verbose=True, log_message=log_message) out, err = capfd.readouterr() - print(out) - print(err) + assert ~log_message.startswith("END") + assert log_message in out def test_SSM_small(get_csr_matrix_small): From 72d63eae4055f4998f8909b16a3b8e650bbef82a Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 19:02:36 +0200 Subject: [PATCH 41/86] micro --- src/flowstab/SparseStochMat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index a604552..2e239e5 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -788,7 +788,7 @@ def reset(self, current_row): self.current_row = current_row self.LS = [] -def csr_add(A,B, use_cython=USE_CYTHON): +def csr_add(A, B, use_cython=USE_CYTHON): """Addition of square csr matrix""" size = A.shape[0] From 989a69398879b89dc6af3f6a7d7f3b4190b203b4 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 19:03:41 +0200 Subject: [PATCH 42/86] started with more tests --- tests/test_sparse_stoch_matrix.py | 39 ++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 5a9c799..9dd57a8 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -31,9 +31,14 @@ def test_SSM_small(get_csr_matrix_small): # ### # inti from scipy.sparse.csr_matrix A_csr = get_csr_matrix_small - for _ in range(10000): - ssm = SSM.from_full_csr_matrix(A_csr) - np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) + ssm = SSM.from_full_csr_matrix(A_csr) + np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) + # crete a diagonal matrix + _ = SSM.create_diag(size=100, diag_val=0.3) + # convert it to a full csr + full_A = A_csr.to_full_mat() + # ... + def test_SSM_large(get_csr_matrix_large): @@ -167,3 +172,31 @@ def test_SSM_inplace_row_normalize_equivalence(get_SSM_matrix_large): np.testing.assert_array_equal(A_ssm2_data, A_ssm2.T_small.data) # test equivalence np.testing.assert_array_equal(A_ssm1.data, A_ssm2.T_small.data) + +def test_csr_operations(get_SSM_matrix_large): + """Check the csr_* funcions + + .. note:: + This function adds csr_matrix objects, so it should be compared to the built in addition + + """ + from flowstab.SparseStochMat import ( + csr_add, + csr_matmul, + csr_csc_matmul, + csr_csrT_matmul, + ) + A_ssm1 = get_SSM_matrix_large + # A_ssm2 = A_ssm1 + 2 + diff_csr = csr_add(A_ssm1, A_ssm1) + np.testing.assert_equal(diff_csr.toarray(), np.full(shape=diff_csr.shape, fill_value=0)) + +def test_SparseAutocovMatrixCSR(): + """Check basic operations on sparse_autocov_csr_mat""" + from flowstab.SparseStochMat import sparse_autocov_csr_mat as SAMCSR + pass + +def test_SparseAutocovMatrix(): + """Check basic operations on sparse_autocov_mat""" + from flowstab.SparseStochMat import sparse_autocov_mat as SAM + pass From d81018d6ba23605bbcfec2c78f59b21129cd1043 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 20:24:35 +0200 Subject: [PATCH 43/86] testing mkl --- .github/workflows/development.yml | 9 +++++++++ tests/conftest.py | 16 ++++++++++++++++ tests/requirements.txt | 1 + tests/test_sparse_stoch_matrix.py | 15 +++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 6ae226d..75c064d 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -10,6 +10,9 @@ jobs: build-venv: runs-on: ubuntu-22.04 steps: + - run: | + sudo apt-get upgrade + sudo apt-get -y install intel-mkl - uses: actions/checkout@v4 - name: Set up Python 3.9 uses: actions/setup-python@v5 @@ -84,6 +87,9 @@ jobs: runs-on: ubuntu-latest needs: install steps: + - run: | + sudo apt-get upgrade + sudo apt-get -y install intel-mkl - uses: actions/checkout@v4 - name: Set up Python 3.9 uses: actions/setup-python@v5 @@ -112,6 +118,9 @@ jobs: runs-on: ubuntu-latest needs: install steps: + - run: | + sudo apt-get upgrade + sudo apt-get -y install intel-mkl - uses: actions/checkout@v4 - name: Set up Python 3.9 uses: actions/setup-python@v5 diff --git a/tests/conftest.py b/tests/conftest.py index aef98c4..ca3d69f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,6 +24,22 @@ def get_csr_matrix_large(): density = nbr_non_zeros / size return csr_matrix((data, (row, col)), shape=(size, size)), density +@pytest.fixture(scope='session') +def get_csr_matrix_pair(): + """Creat an exemplary csr matrix that can be used for testing + """ + size = 10000 + nbr_non_zeros = 1000 + row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + data = np.random.randint(0,100, size=nbr_non_zeros) + A = csr_matrix((data, (row, col)), shape=(size, size)) + row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + data = np.random.randint(0,100, size=nbr_non_zeros) + B = csr_matrix((data, (row, col)), shape=(size, size)) + return A, B + @pytest.fixture(scope='function') def get_SSM_matrix_large(): """Creat an exemplary csr matrix that can be used for testing diff --git a/tests/requirements.txt b/tests/requirements.txt index 86d8246..f89a46c 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -2,3 +2,4 @@ pytest>=8.3.2 ruff>=0.5.7 pytest-cov==5.0.0 pytest-memray==1.7.0 +sparse_dot_mkl==0.9.4 diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 9dd57a8..0dffe57 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -200,3 +200,18 @@ def test_SparseAutocovMatrix(): """Check basic operations on sparse_autocov_mat""" from flowstab.SparseStochMat import sparse_autocov_mat as SAM pass + + +def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): + """ + """ + from sparse_dot_mkl import dot_product_mkl as mkl_matmul + A, B = get_csr_matrix_pair + mkl_matmul(A, B) + +def test_sparse_matmul_memory(): + """ + """ + from sparse_dot_mkl import dot_product_mkl as mkl_matmul + A, B = get_csr_matrix_pair + mkl_matmul(A, B) From 162b757e0fec7564f5a4800117d32adc6131bf61 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 21:44:56 +0200 Subject: [PATCH 44/86] float matrices --- tests/conftest.py | 4 ++-- tests/test_sparse_stoch_matrix.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ca3d69f..a0783d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,11 +32,11 @@ def get_csr_matrix_pair(): nbr_non_zeros = 1000 row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - data = np.random.randint(0,100, size=nbr_non_zeros) + data = np.random.random(size=nbr_non_zeros) A = csr_matrix((data, (row, col)), shape=(size, size)) row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - data = np.random.randint(0,100, size=nbr_non_zeros) + data = np.random.random(size=nbr_non_zeros) B = csr_matrix((data, (row, col)), shape=(size, size)) return A, B diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 0dffe57..271608a 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -209,7 +209,7 @@ def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): A, B = get_csr_matrix_pair mkl_matmul(A, B) -def test_sparse_matmul_memory(): +def test_sparse_matmul_memory(get_csr_matrix_pair): """ """ from sparse_dot_mkl import dot_product_mkl as mkl_matmul From 6100fe7c81497033a1f97bdd755356e71ba31554 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 22:11:09 +0200 Subject: [PATCH 45/86] matmul testing --- tests/conftest.py | 4 ++-- tests/test_sparse_stoch_matrix.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a0783d2..21c5dc8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,8 +28,8 @@ def get_csr_matrix_large(): def get_csr_matrix_pair(): """Creat an exemplary csr matrix that can be used for testing """ - size = 10000 - nbr_non_zeros = 1000 + size = 100000 + nbr_non_zeros = 5000 row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) data = np.random.random(size=nbr_non_zeros) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 271608a..d1b8cd1 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -201,17 +201,17 @@ def test_SparseAutocovMatrix(): from flowstab.SparseStochMat import sparse_autocov_mat as SAM pass - -def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): +def test_sparse_matmul_memory(get_csr_matrix_pair): """ """ - from sparse_dot_mkl import dot_product_mkl as mkl_matmul A, B = get_csr_matrix_pair - mkl_matmul(A, B) + for _ in range(5000): + _ = A @ B -def test_sparse_matmul_memory(get_csr_matrix_pair): +def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): """ """ from sparse_dot_mkl import dot_product_mkl as mkl_matmul A, B = get_csr_matrix_pair - mkl_matmul(A, B) + for _ in range(5000): + _ = mkl_matmul(A, B) From 44c7551894f2da7ef12b000d1943fc22631830f0 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 2 Sep 2024 22:40:57 +0200 Subject: [PATCH 46/86] testing with larger matrices --- tests/conftest.py | 4 ++-- tests/test_sparse_stoch_matrix.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 21c5dc8..22d5364 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,8 +28,8 @@ def get_csr_matrix_large(): def get_csr_matrix_pair(): """Creat an exemplary csr matrix that can be used for testing """ - size = 100000 - nbr_non_zeros = 5000 + size = 10000000 + nbr_non_zeros = 100000 row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) data = np.random.random(size=nbr_non_zeros) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index d1b8cd1..846934b 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -201,17 +201,17 @@ def test_SparseAutocovMatrix(): from flowstab.SparseStochMat import sparse_autocov_mat as SAM pass -def test_sparse_matmul_memory(get_csr_matrix_pair): +def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): """ """ + from sparse_dot_mkl import dot_product_mkl as mkl_matmul A, B = get_csr_matrix_pair - for _ in range(5000): - _ = A @ B + for _ in range(1000): + _ = mkl_matmul(A, B) -def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): +def test_sparse_matmul_memory(get_csr_matrix_pair): """ """ - from sparse_dot_mkl import dot_product_mkl as mkl_matmul A, B = get_csr_matrix_pair - for _ in range(5000): - _ = mkl_matmul(A, B) + for _ in range(1000): + _ = A @ B From d3b90d27449228eeb828aff37b3dad4d88eea518 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 3 Sep 2024 11:17:20 +0200 Subject: [PATCH 47/86] testing mkl and blas/lapak --- .github/workflows/development.yml | 26 +++++++++++++++-------- pyproject.toml | 11 +++++++--- requirements.txt => requirements/base.txt | 2 +- requirements/mkl.txt | 3 +++ 4 files changed, 29 insertions(+), 13 deletions(-) rename requirements.txt => requirements/base.txt (80%) create mode 100644 requirements/mkl.txt diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index 75c064d..c72856e 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -23,11 +23,13 @@ jobs: uses: actions/cache/restore@v4 with: path: ./venv - key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/**.toml') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/requirements/base.txt') }}-${{ hashFiles('**/**.toml') }} - name: Build venv run: | python -m pip install --upgrade pip python -m venv ./venv + # with mkl support + ./venv/bin/python -m pip install -r requirements/mkl.txt ./venv/bin/python -m pip install .[testing] if: steps.cached-venv.outputs.cache-hit != 'true' - name: Cache venv @@ -47,7 +49,7 @@ jobs: uses: actions/cache/restore@v4 with: path: ./venv - key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/**.toml') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/requirements/base.txt') }}-${{ hashFiles('**/**.toml') }} - name: Build wheel run: | venv/bin/python -m pip install build @@ -67,7 +69,7 @@ jobs: - uses: actions/cache/restore@v4 with: path: ./venv - key: venv-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/**.toml') }} + key: venv-${{ runner.os }}-${{ hashFiles('**/requirements/base.txt') }}-${{ hashFiles('**/**.toml') }} - name: Getting build wheel uses: actions/download-artifact@v4 with: @@ -126,12 +128,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.9 - # Note: we use cache here since artifacts do not keep permissions - - name: Getting the venv - uses: actions/cache/restore@v4 - with: - path: ./venv - key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} + - name: Build venv + run: | + python -m pip install --upgrade pip + python -m venv ./venv + # with mkl support + ./venv/bin/python -m pip install .[testing,mkl] + # # Note: we use cache here since artifacts do not keep permissions + # - name: Getting the venv + # uses: actions/cache/restore@v4 + # with: + # path: ./venv + # key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray -k '_memory' diff --git a/pyproject.toml b/pyproject.toml index b604ca7..081ab99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,14 @@ Repository = "https://github.com/alexbovet/flow_stability.git" Documentation = "https://flow-stability.readthedocs.io/" Issues = "https://github.com/alexbovet/flow_stability/issues" -[tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} -readme = {file = ["README.md"]} +[tool.setuptools.dynamic.readme] +file = ["README.md"] + +[tool.setuptools.dynamic.dependencies] +file = ["requirements/base.txt"] + +[tool.setuptools.dynamic.optional-dependencies.mkl] +file = ["requirements/mkl.txt"] [tool.setuptools.dynamic.optional-dependencies.testing] file = ["tests/requirements.txt"] diff --git a/requirements.txt b/requirements/base.txt similarity index 80% rename from requirements.txt rename to requirements/base.txt index 63e77f5..483f38b 100644 --- a/requirements.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ Cython>=3.0.11 pandas>=2.2.2 -scipy>=1.13.1 hickle>=5.0.3 psutil>=6.0.0 +scipy>=1.10 diff --git a/requirements/mkl.txt b/requirements/mkl.txt new file mode 100644 index 0000000..e7a6c70 --- /dev/null +++ b/requirements/mkl.txt @@ -0,0 +1,3 @@ +intel-scipy +# -i https://pypi.anaconda.org/intel/simple +# scipy>=1.10 From 117faf38f3219b4062a756ad99c4540097994bcf Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sat, 14 Sep 2024 18:05:51 +0200 Subject: [PATCH 48/86] wip on profiling and testing csr ops --- tests/conftest.py | 47 ++++++++------ tests/test_sparse_stoch_matrix.py | 103 ++++++++++++++++++++++++++---- 2 files changed, 120 insertions(+), 30 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 22d5364..1f9c8bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,32 +25,41 @@ def get_csr_matrix_large(): return csr_matrix((data, (row, col)), shape=(size, size)), density @pytest.fixture(scope='session') -def get_csr_matrix_pair(): +def csr_matrix_creator(): """Creat an exemplary csr matrix that can be used for testing """ size = 10000000 nbr_non_zeros = 100000 - row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - data = np.random.random(size=nbr_non_zeros) - A = csr_matrix((data, (row, col)), shape=(size, size)) - row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - data = np.random.random(size=nbr_non_zeros) - B = csr_matrix((data, (row, col)), shape=(size, size)) - return A, B -@pytest.fixture(scope='function') -def get_SSM_matrix_large(): + def _get_matrix(nbr:int=1,size:int=size, nbr_non_zeros:int=nbr_non_zeros): + matrices = [] + for _ in range(nbr): + row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + data = np.random.random(size=nbr_non_zeros) + a_csr = csr_matrix((data, (row, col)), shape=(size, size)) + a_csr.indptr = a_csr.indptr.astype(np.int32, copy=False) + a_csr.indices = a_csr.indices.astype(np.int32, copy=False) + matrices.append(a_csr) + return tuple(matrices) + return _get_matrix + +@pytest.fixture(scope='session') +def SSM_matrix_creator(): """Creat an exemplary csr matrix that can be used for testing """ from flowstab.SparseStochMat import sparse_stoch_mat size = 1000000 nbr_non_zeros = 1000 - row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) - data = np.random.randint(0,1, size=nbr_non_zeros) - _a_csr = csr_matrix((data, (row, col)), shape=(size, size)) - _a_csr.indptr = _a_csr.indptr.astype(np.int64, copy=False) - _a_csr.indices = _a_csr.indices.astype(np.int64, copy=False) - return sparse_stoch_mat.from_full_csr_matrix(_a_csr) + def _get_matrix(nbr:int=1,size:int=size, nbr_non_zeros:int=nbr_non_zeros): + matrices = [] + for _ in range(nbr): + row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) + data = np.random.randint(0,1, size=nbr_non_zeros) + _a_csr = csr_matrix((data, (row, col)), shape=(size, size)) + _a_csr.indptr = _a_csr.indptr.astype(np.int32, copy=False) + _a_csr.indices = _a_csr.indices.astype(np.int64, copy=False) + matrices.append(sparse_stoch_mat.from_full_csr_matrix(_a_csr)) + return tuple(matrices) + return _get_matrix diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 846934b..716c319 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -150,7 +150,7 @@ def test_SSM_from_full_csr_equivalence(get_csr_matrix_large): np.testing.assert_array_equal(nc_indptr, c_indptr) np.testing.assert_array_equal(nc_nz_rowcols, c_nz_rowcols) -def test_SSM_inplace_row_normalize_equivalence(get_SSM_matrix_large): +def test_SSM_inplace_row_normalize_equivalence(SSM_matrix_creator): """Make sure the cython and pure python implementations are equivalent """ from flowstab.SparseStochMat import ( @@ -159,7 +159,7 @@ def test_SSM_inplace_row_normalize_equivalence(get_SSM_matrix_large): from flowstab._cython_sparse_stoch_subst import ( inplace_csr_row_normalize ) - A_ssm1 = get_SSM_matrix_large + A_ssm1 = SSM_matrix_creator(nbr=1)[0] A_ssm1_data = copy(A_ssm1.T_small.data) A_ssm2 = copy(A_ssm1) A_ssm2_data = copy(A_ssm2.T_small.data) @@ -173,7 +173,11 @@ def test_SSM_inplace_row_normalize_equivalence(get_SSM_matrix_large): # test equivalence np.testing.assert_array_equal(A_ssm1.data, A_ssm2.T_small.data) -def test_csr_operations(get_SSM_matrix_large): +# ### +# Testing the csr operations +# ### + +def test_csr_add_compare(csr_matrix_creator): """Check the csr_* funcions .. note:: @@ -186,10 +190,87 @@ def test_csr_operations(get_SSM_matrix_large): csr_csc_matmul, csr_csrT_matmul, ) - A_ssm1 = get_SSM_matrix_large - # A_ssm2 = A_ssm1 + 2 - diff_csr = csr_add(A_ssm1, A_ssm1) - np.testing.assert_equal(diff_csr.toarray(), np.full(shape=diff_csr.shape, fill_value=0)) + A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=100000, nbr_non_zeros=2000) + sum_csr_native = A_ssm1 + A_ssm2 + sum_csr = csr_add(A_ssm1, A_ssm2) + np.testing.assert_equal(sum_csr_native.indices, sum_csr.indices) + np.testing.assert_equal(sum_csr_native.indptr, sum_csr.indptr) + np.testing.assert_allclose(sum_csr_native.data, sum_csr.data) + +def test_csr_add_memory(csr_matrix_creator): + """Check the csr_* funcions + + .. note:: + This function adds csr_matrix objects, so it should be compared to the built in addition + + """ + from flowstab.SparseStochMat import ( + csr_add, + csr_matmul, + csr_csc_matmul, + csr_csrT_matmul, + ) + A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=100000, nbr_non_zeros=2000) + _ = csr_add(A_ssm1, A_ssm2) + +def test_csr_add_native_memory(csr_matrix_creator): + """Check the csr_* funcions + + .. note:: + This function adds csr_matrix objects, so it should be compared to the built in addition + + """ + from flowstab.SparseStochMat import ( + csr_add, + csr_matmul, + csr_csc_matmul, + csr_csrT_matmul, + ) + A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=100000, nbr_non_zeros=2000) + _ = A_ssm1 + A_ssm2 + +def test_csr_matmul_compare(csr_matrix_creator): + """Check the csr_* funcions + + .. note:: + This function adds csr_matrix objects, so it should be compared to the built in addition + + """ + from flowstab.SparseStochMat import ( + csr_matmul, + ) + A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=10000000, nbr_non_zeros=200000) + mmul_csr_native = A_ssm1 @ A_ssm2 + mmul_csr = csr_matmul(A_ssm1, A_ssm2) + np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) + np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) + +def test_csr_matmul_native_memory(csr_matrix_creator): + """Check the csr_* funcions + + .. note:: + This function adds csr_matrix objects, so it should be compared to the built in addition + + """ + from flowstab.SparseStochMat import ( + csr_matmul, + ) + A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=10000000, nbr_non_zeros=200000) + _ = A_ssm1 @ A_ssm2 + +def test_csr_matmul_memory(csr_matrix_creator): + """Check the csr_* funcions + + .. note:: + This function adds csr_matrix objects, so it should be compared to the built in addition + + """ + from flowstab.SparseStochMat import ( + csr_matmul, + ) + A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=10000000, nbr_non_zeros=200000) + _ = csr_matmul(A_ssm1, A_ssm2) def test_SparseAutocovMatrixCSR(): """Check basic operations on sparse_autocov_csr_mat""" @@ -201,17 +282,17 @@ def test_SparseAutocovMatrix(): from flowstab.SparseStochMat import sparse_autocov_mat as SAM pass -def test_sparse_matmul_mkl_memory(get_csr_matrix_pair): +def test_sparse_matmul_mkl_memory(csr_matrix_creator): """ """ from sparse_dot_mkl import dot_product_mkl as mkl_matmul - A, B = get_csr_matrix_pair + A, B = csr_matrix_creator(nbr=2) for _ in range(1000): _ = mkl_matmul(A, B) -def test_sparse_matmul_memory(get_csr_matrix_pair): +def test_sparse_matmul_memory(csr_matrix_creator): """ """ - A, B = get_csr_matrix_pair + A, B = csr_matrix_creator(nbr=2) for _ in range(1000): _ = A @ B From ee5e48f3e75eabec91389d90e3cf1cb60e25f282 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 15 Sep 2024 00:11:39 +0200 Subject: [PATCH 49/86] completing tests for csr methods --- tests/conftest.py | 18 ++-- tests/test_sparse_stoch_matrix.py | 153 +++++++++++++++--------------- 2 files changed, 90 insertions(+), 81 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1f9c8bd..e796dd7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ import pytest import numpy as np -from scipy.sparse import csr_matrix +from scipy.sparse import csr_matrix, csc_matrix @pytest.fixture(scope='function') def get_csr_matrix_small(): @@ -25,19 +25,25 @@ def get_csr_matrix_large(): return csr_matrix((data, (row, col)), shape=(size, size)), density @pytest.fixture(scope='session') -def csr_matrix_creator(): +def cs_matrix_creator(): """Creat an exemplary csr matrix that can be used for testing """ - size = 10000000 - nbr_non_zeros = 100000 + size = 1000000 + nbr_non_zeros = 10000 - def _get_matrix(nbr:int=1,size:int=size, nbr_non_zeros:int=nbr_non_zeros): + def _get_matrix(nbr:int=1,size:int=size, nbr_non_zeros:int=nbr_non_zeros, mode='r'): matrices = [] + assert mode in ['r', 'c'] + if mode == 'r': + matrix_gen = csr_matrix + else: + matrix_gen = csc_matrix + for _ in range(nbr): row = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) col = np.sort(np.random.randint(0, size, size=nbr_non_zeros)) data = np.random.random(size=nbr_non_zeros) - a_csr = csr_matrix((data, (row, col)), shape=(size, size)) + a_csr = matrix_gen((data, (row, col)), shape=(size, size)) a_csr.indptr = a_csr.indptr.astype(np.int32, copy=False) a_csr.indices = a_csr.indices.astype(np.int32, copy=False) matrices.append(a_csr) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 716c319..570667b 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -7,6 +7,14 @@ eye, ) +from flowstab.SparseStochMat import ( + csr_add, + csr_matmul, + csr_csc_matmul, + csr_csrT_matmul, +) + + def test_timing(capfd): """Bacic operations with the 'spares_stoch_mat' class """ @@ -177,100 +185,95 @@ def test_SSM_inplace_row_normalize_equivalence(SSM_matrix_creator): # Testing the csr operations # ### -def test_csr_add_compare(csr_matrix_creator): - """Check the csr_* funcions - - .. note:: - This function adds csr_matrix objects, so it should be compared to the built in addition - - """ - from flowstab.SparseStochMat import ( - csr_add, - csr_matmul, - csr_csc_matmul, - csr_csrT_matmul, - ) - A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=100000, nbr_non_zeros=2000) +def test_csr_add_compare(cs_matrix_creator): + """Compare the csr_add to a native implementation""" + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=1000000, + nbr_non_zeros=20000, mode='r') sum_csr_native = A_ssm1 + A_ssm2 sum_csr = csr_add(A_ssm1, A_ssm2) + np.testing.assert_allclose(sum_csr_native.data, sum_csr.data) np.testing.assert_equal(sum_csr_native.indices, sum_csr.indices) np.testing.assert_equal(sum_csr_native.indptr, sum_csr.indptr) - np.testing.assert_allclose(sum_csr_native.data, sum_csr.data) - -def test_csr_add_memory(csr_matrix_creator): - """Check the csr_* funcions - - .. note:: - This function adds csr_matrix objects, so it should be compared to the built in addition - """ - from flowstab.SparseStochMat import ( - csr_add, - csr_matmul, - csr_csc_matmul, - csr_csrT_matmul, - ) - A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=100000, nbr_non_zeros=2000) +def test_csr_add_memory(cs_matrix_creator): + """Check the csr_add function for timing and memory consumption""" + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=1000000, + nbr_non_zeros=20000, mode='r') _ = csr_add(A_ssm1, A_ssm2) -def test_csr_add_native_memory(csr_matrix_creator): - """Check the csr_* funcions - - .. note:: - This function adds csr_matrix objects, so it should be compared to the built in addition - - """ - from flowstab.SparseStochMat import ( - csr_add, - csr_matmul, - csr_csc_matmul, - csr_csrT_matmul, - ) - A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=100000, nbr_non_zeros=2000) +def test_csr_add_native_memory(cs_matrix_creator): + """Check the csr native addition for timing and memory consumption""" + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=1000000, + nbr_non_zeros=20000, mode='r') _ = A_ssm1 + A_ssm2 -def test_csr_matmul_compare(csr_matrix_creator): - """Check the csr_* funcions - - .. note:: - This function adds csr_matrix objects, so it should be compared to the built in addition - - """ - from flowstab.SparseStochMat import ( - csr_matmul, - ) - A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=10000000, nbr_non_zeros=200000) +def test_csr_matmul_compare(cs_matrix_creator): + """Compare the csr_matmul to a native implementation""" + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000000, + nbr_non_zeros=200000, mode='r') mmul_csr_native = A_ssm1 @ A_ssm2 mmul_csr = csr_matmul(A_ssm1, A_ssm2) + np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) - np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) -def test_csr_matmul_native_memory(csr_matrix_creator): - """Check the csr_* funcions +def test_csr_matmul_native_memory(cs_matrix_creator): + """Check the csr_matmul function for timing and memory consumption""" + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000000, + nbr_non_zeros=200000, mode='r') + _ = A_ssm1 @ A_ssm2 - .. note:: - This function adds csr_matrix objects, so it should be compared to the built in addition +def test_csr_matmul_memory(cs_matrix_creator): + """Check the csr native @ function for timing and memory consumption""" + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000000, + nbr_non_zeros=200000, mode='r') + _ = csr_matmul(A_ssm1, A_ssm2) - """ - from flowstab.SparseStochMat import ( - csr_matmul, - ) - A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=10000000, nbr_non_zeros=200000) - _ = A_ssm1 @ A_ssm2 +def test_csr_csc_matmul_compare(cs_matrix_creator): + """Compare the csr_csc_matmul to a native implementation""" + A_csr, = cs_matrix_creator(nbr=1, mode='r') + A_csc, = cs_matrix_creator(nbr=1, mode='c') + mmul_csr_native = A_csr @ A_csc.tocsr() + mmul_csr = csr_csc_matmul(A_csr, A_csc) + np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) + np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) + np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) -def test_csr_matmul_memory(csr_matrix_creator): - """Check the csr_* funcions +def test_csr_csc_matmul_native_memory(cs_matrix_creator): + """Check the csr_matmul function for timing and memory consumption""" + A_csr, = cs_matrix_creator(nbr=1, mode='r') + A_csc, = cs_matrix_creator(nbr=1, mode='c') + _ = A_csr @ A_csc.tocsr() + +def test_csr_csc_matmul_memory(cs_matrix_creator): + """Check the csr native @ function for timing and memory consumption""" + A_csr, = cs_matrix_creator(nbr=1, mode='r') + A_csc, = cs_matrix_creator(nbr=1, mode='c') + _ = csr_csc_matmul(A_csr, A_csc) + +def test_csr_csrT_matmul_compare(cs_matrix_creator): + """Compare the csr_csrT_matmul to a native implementation""" + A_csr1, = cs_matrix_creator(nbr=1, mode='r') + A_csr2, = cs_matrix_creator(nbr=1, mode='c') + mmul_csr_native = A_csr1 @ A_csr2.tocsr().T + mmul_csr = csr_csrT_matmul(A_csr1, A_csr2) + np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) + np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) + np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) - .. note:: - This function adds csr_matrix objects, so it should be compared to the built in addition +def test_csr_csrT_matmul_native_memory(cs_matrix_creator): + """Check the csr_csrT_matmul function for timing and memory consumption""" + A_csr, = cs_matrix_creator(nbr=1, mode='r') + A_csc, = cs_matrix_creator(nbr=1, mode='c') + _ = A_csr @ A_csc.tocsr().T - """ - from flowstab.SparseStochMat import ( - csr_matmul, - ) - A_ssm1, A_ssm2 = csr_matrix_creator(nbr=2, size=10000000, nbr_non_zeros=200000) - _ = csr_matmul(A_ssm1, A_ssm2) +def test_csr_csrT_matmul_memory(cs_matrix_creator): + """Check the csr_csrT native @ function for timing and memory consumption""" + A_csr, = cs_matrix_creator(nbr=1, mode='r') + A_csc, = cs_matrix_creator(nbr=1, mode='c') + _ = csr_csrT_matmul(A_csr, A_csc) + +# ### def test_SparseAutocovMatrixCSR(): """Check basic operations on sparse_autocov_csr_mat""" From bef73bead965c406750d6250ea1bf9e4bdefc4ff Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 15 Sep 2024 14:11:43 +0200 Subject: [PATCH 50/86] testing with comparison crs_ operations --- tests/conftest.py | 18 ++++++++++ tests/test_sparse_stoch_matrix.py | 58 ++++++++++++++++++------------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e796dd7..694bab2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,3 +69,21 @@ def _get_matrix(nbr:int=1,size:int=size, nbr_non_zeros:int=nbr_non_zeros): matrices.append(sparse_stoch_mat.from_full_csr_matrix(_a_csr)) return tuple(matrices) return _get_matrix + +@pytest.fixture(scope='session') +def compare_alike(): + def compare_sparse_matrice(A, B): + """Checks if two csr matrices describe the same matrix + + csr notation can deviate in that data and indices can be re-arranged + within a indptr slice. + """ + assert len(A.indptr) == len(B.indptr) + for i in range(len(A.indptr) - 1): + A_s, A_e = A.indptr[i:i+2] + B_s, B_e = B.indptr[i:i+2] + B_sorted = B.indices[B_s: B_e].argsort() + A_sorted = A.indices[A_s: A_e].argsort() + np.testing.assert_equal(A.indices[A_s:A_e][A_sorted], B.indices[B_s:B_e][B_sorted]) + np.testing.assert_equal(B.data[B_s:B_e][B_sorted], A.data[A_s:A_e][A_sorted]) + return compare_sparse_matrice diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 570667b..41d5a0a 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -185,15 +185,16 @@ def test_SSM_inplace_row_normalize_equivalence(SSM_matrix_creator): # Testing the csr operations # ### -def test_csr_add_compare(cs_matrix_creator): +def test_csr_add_compare(cs_matrix_creator, compare_alike): """Compare the csr_add to a native implementation""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=1000000, - nbr_non_zeros=20000, mode='r') + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000, + nbr_non_zeros=500, mode='r') sum_csr_native = A_ssm1 + A_ssm2 sum_csr = csr_add(A_ssm1, A_ssm2) - np.testing.assert_allclose(sum_csr_native.data, sum_csr.data) - np.testing.assert_equal(sum_csr_native.indices, sum_csr.indices) - np.testing.assert_equal(sum_csr_native.indptr, sum_csr.indptr) + # np.testing.assert_allclose(sum_csr_native.data, sum_csr.data) + # np.testing.assert_equal(sum_csr_native.indices, sum_csr.indices) + # np.testing.assert_equal(sum_csr_native.indptr, sum_csr.indptr) + compare_alike(sum_csr, sum_csr_native) def test_csr_add_memory(cs_matrix_creator): """Check the csr_add function for timing and memory consumption""" @@ -207,15 +208,16 @@ def test_csr_add_native_memory(cs_matrix_creator): nbr_non_zeros=20000, mode='r') _ = A_ssm1 + A_ssm2 -def test_csr_matmul_compare(cs_matrix_creator): +def test_csr_matmul_compare(cs_matrix_creator, compare_alike): """Compare the csr_matmul to a native implementation""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000000, - nbr_non_zeros=200000, mode='r') + A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=100000, + nbr_non_zeros=2000, mode='r') mmul_csr_native = A_ssm1 @ A_ssm2 mmul_csr = csr_matmul(A_ssm1, A_ssm2) - np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) - np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) - np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) + # np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) + # np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + compare_alike(mmul_csr_native, mmul_csr) def test_csr_matmul_native_memory(cs_matrix_creator): """Check the csr_matmul function for timing and memory consumption""" @@ -229,15 +231,18 @@ def test_csr_matmul_memory(cs_matrix_creator): nbr_non_zeros=200000, mode='r') _ = csr_matmul(A_ssm1, A_ssm2) -def test_csr_csc_matmul_compare(cs_matrix_creator): +def test_csr_csc_matmul_compare(cs_matrix_creator, compare_alike): """Compare the csr_csc_matmul to a native implementation""" - A_csr, = cs_matrix_creator(nbr=1, mode='r') - A_csc, = cs_matrix_creator(nbr=1, mode='c') + A_csr, = cs_matrix_creator(nbr=1, size=100000, + nbr_non_zeros=2000, mode='r') + A_csc, = cs_matrix_creator(nbr=1, size=100000, + nbr_non_zeros=2000, mode='c') mmul_csr_native = A_csr @ A_csc.tocsr() mmul_csr = csr_csc_matmul(A_csr, A_csc) - np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) - np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) - np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) + # np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) + # np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + compare_alike(mmul_csr_native, mmul_csr) def test_csr_csc_matmul_native_memory(cs_matrix_creator): """Check the csr_matmul function for timing and memory consumption""" @@ -251,15 +256,18 @@ def test_csr_csc_matmul_memory(cs_matrix_creator): A_csc, = cs_matrix_creator(nbr=1, mode='c') _ = csr_csc_matmul(A_csr, A_csc) -def test_csr_csrT_matmul_compare(cs_matrix_creator): +def test_csr_csrT_matmul_compare(cs_matrix_creator, compare_alike): """Compare the csr_csrT_matmul to a native implementation""" - A_csr1, = cs_matrix_creator(nbr=1, mode='r') - A_csr2, = cs_matrix_creator(nbr=1, mode='c') - mmul_csr_native = A_csr1 @ A_csr2.tocsr().T + A_csr1, = cs_matrix_creator(nbr=1, size=100000, + nbr_non_zeros=2000, mode='r') + A_csr2, = cs_matrix_creator(nbr=1, size=100000, + nbr_non_zeros=2000,mode='r') + mmul_csr_native = A_csr1 @ A_csr2.T mmul_csr = csr_csrT_matmul(A_csr1, A_csr2) - np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) - np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) - np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) + # np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) + # np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) + compare_alike(mmul_csr_native, mmul_csr) def test_csr_csrT_matmul_native_memory(cs_matrix_creator): """Check the csr_csrT_matmul function for timing and memory consumption""" From 50fb6a7f6fb8a533c21c99dd503b18d0b7206ffb Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 15 Sep 2024 22:40:50 +0200 Subject: [PATCH 51/86] show all memory reports --- .github/workflows/development.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index c72856e..a5b2907 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -142,7 +142,7 @@ jobs: # key: venv-${{ runner.os }}-${{ hashFiles('venv/**') }} - name: Tests with memory tracking run: | - venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray -k '_memory' + venv/bin/python -m pytest --junitxml=junit/test-results.xml --cov=flowstab --durations=0 --memray --most-allocations=0 -k '_memory' env: COVERAGE_FILE: ".coverage.memory" - name: Store coverage file From 62fe6be960c3969ca703dfb3a7b8120c243b3f8c Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 00:09:14 +0200 Subject: [PATCH 52/86] covering conversion from/to ssm --- tests/test_sparse_stoch_matrix.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 41d5a0a..5cf9b0f 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -181,8 +181,16 @@ def test_SSM_inplace_row_normalize_equivalence(SSM_matrix_creator): # test equivalence np.testing.assert_array_equal(A_ssm1.data, A_ssm2.T_small.data) -# ### -# Testing the csr operations +def test_rebuild_nnz_rowcol(cs_matrix_creator, compare_alike): + """Test conversions from ssm to csr and back + """ + from flowstab.SparseStochMat import sparse_stoch_mat as SSM + A_csr = cs_matrix_creator(nbr=1, size=100000, nbr_non_zeros=1000)[0] + A_ssm = SSM.from_full_csr_matrix(Tcsr=A_csr) + A_rebuild = A_ssm.to_full_mat() + compare_alike(A_csr, A_rebuild) + +# ### Testing the csr operations # ### def test_csr_add_compare(cs_matrix_creator, compare_alike): @@ -261,7 +269,7 @@ def test_csr_csrT_matmul_compare(cs_matrix_creator, compare_alike): A_csr1, = cs_matrix_creator(nbr=1, size=100000, nbr_non_zeros=2000, mode='r') A_csr2, = cs_matrix_creator(nbr=1, size=100000, - nbr_non_zeros=2000,mode='r') + nbr_non_zeros=2000, mode='r') mmul_csr_native = A_csr1 @ A_csr2.T mmul_csr = csr_csrT_matmul(A_csr1, A_csr2) # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) From a27f9f45d0f4d0a0c6d97c316003a45a0adf8e5c Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 14:32:04 +0200 Subject: [PATCH 53/86] formatting docstring - correcting description --- src/flowstab/SparseStochMat.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 2e239e5..901908b 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -32,6 +32,7 @@ from scipy.sparse import ( coo_matrix, csr_matrix, + csc_matrix, eye, isspmatrix_csc, isspmatrix_csr, @@ -1166,10 +1167,11 @@ def rebuild_nnz_rowcol(T_small:csr_matrix, nonzero_indices:NDArray, def inplace_csr_matmul_diag(A, diag_vec): - """Inplace multiply a csr matrix A with a diag matrix D = diag(diag_vec): + """Inplace multiply a csr matrix A with a diag matrix D A = A @ D - + + With D = np.diagflat(diag_vec) and A a scipy.sparse.cs[rc]_matrix, i.e. column i of A is scaled by diag_vec[i] """ @@ -1196,11 +1198,12 @@ def inplace_csr_matmul_diag(A, diag_vec): -def inplace_diag_matmul_csr(A, diag_vec): - """Inplace multiply a diag matrix D = diag(diag_vec) with a csr matrix A: +def inplace_diag_matmul_csr(A:csr_matrix | csc_matrix, diag_vec: NDArray)->None: + """Inplace multiply a diag matrix D with a csr matrix A: A = D @ A + With D = np.diagflat(diag_vec) and A a scipy.sparse.cs[rc]_matrix, i.e. row i of A is scaled by diag_vec[i] """ From 28b405144549abdcf81962042161ee7bc2d94c07 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 14:32:50 +0200 Subject: [PATCH 54/86] temp. fix for type inconsistency - see #43 --- src/flowstab/SparseStochMat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 901908b..0dc41e7 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -220,7 +220,7 @@ def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, return cls(*_css.sparse_stoch_from_full_csr( np.array(nz_rowcols, dtype=np.int32), Tcsr_data, - Tcsr.indices, + Tcsr.indices.astype(np.int64), Tcsr.indptr, diag_val )) From e0b27269cebb85f7eada8190a4125acf65b57158 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 14:33:14 +0200 Subject: [PATCH 55/86] minor formatting --- src/flowstab/SparseStochMat.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 0dc41e7..753b44e 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -749,11 +749,6 @@ def __rmul__(self, o): return self.__mul__(o) - - - - - class SPA: """sparse accumulator with multiple switch technique @@ -789,6 +784,7 @@ def reset(self, current_row): self.current_row = current_row self.LS = [] + def csr_add(A, B, use_cython=USE_CYTHON): """Addition of square csr matrix""" size = A.shape[0] @@ -2011,7 +2007,7 @@ def sparse_outer(p, use_mkl=True, triu=True, verbose=False, log_message=""): @timing -def sparse_matmul(A,B, verbose=False, log_message=""): +def sparse_matmul(A, B, verbose=False, log_message=""): """Sparse matrix multiplication. Uses sparse_dot_mkl if available, otherwise scipy sparse """ From b658f27a67921b5f69d192ebbe8c058c8cfea73d Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 14:33:41 +0200 Subject: [PATCH 56/86] tests for inplace ops with diag matrix --- tests/test_sparse_stoch_matrix.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 5cf9b0f..413b59f 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -291,6 +291,38 @@ def test_csr_csrT_matmul_memory(cs_matrix_creator): # ### +def test_inplace_diag_matmul_csr(cs_matrix_creator): + """Check the inplace diagonal multiplication for csr and csc""" + from flowstab.SparseStochMat import ( + inplace_csr_matmul_diag, + inplace_diag_matmul_csr + ) + size = 1000 + nnz = 100 + A_csr, = cs_matrix_creator(nbr=1, size=size, nbr_non_zeros=nnz, mode='r') + A_csc, = cs_matrix_creator(nbr=1, size=size, nbr_non_zeros=nnz, mode='c') + Acsr_array = A_csr.toarray() + Acsc_array = A_csc.toarray() + diag_array = np.random.randint(0, 10, size=size) + # test the csr sparse matrix column resacling + inplace_csr_matmul_diag(A_csr, diag_array) + Diag = np.diagflat(diag_array) + Acsr_rescaled = Acsr_array @ Diag + np.testing.assert_equal(A_csr.toarray(), Acsr_rescaled) + # now rescale the rows + inplace_diag_matmul_csr(A_csr, diag_array) + Acsr_rescaled_row = Diag @ Acsr_rescaled + np.testing.assert_equal(A_csr.toarray(), Acsr_rescaled_row) + # test the csc sparse matrix column rescaling + inplace_csr_matmul_diag(A_csc, diag_array) + Acsc_rescaled = Acsc_array @ Diag + np.testing.assert_equal(A_csc.toarray(), Acsc_rescaled) + # now rescale the rows + inplace_diag_matmul_csr(A_csc, diag_array) + Acsc_rescaled_row = Diag @ Acsc_rescaled + np.testing.assert_equal(A_csc.toarray(), Acsc_rescaled_row) + + def test_SparseAutocovMatrixCSR(): """Check basic operations on sparse_autocov_csr_mat""" from flowstab.SparseStochMat import sparse_autocov_csr_mat as SAMCSR From 02a2e873cddf2f943441fabe6b97cb153ad6b015 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 14:52:33 +0200 Subject: [PATCH 57/86] minor correction --- tests/test_sparse_stoch_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 413b59f..f4628db 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -44,7 +44,7 @@ def test_SSM_small(get_csr_matrix_small): # crete a diagonal matrix _ = SSM.create_diag(size=100, diag_val=0.3) # convert it to a full csr - full_A = A_csr.to_full_mat() + full_A = ssm.to_full_mat() # ... From db16b602c0506eeb398033e0ee158696ac166895 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 19:35:46 +0200 Subject: [PATCH 58/86] bettery typeing --- src/flowstab/SparseStochMat.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 753b44e..ea270ce 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -27,6 +27,7 @@ from copy import copy from functools import wraps +from typing import Union import numpy as np from numpy.typing import NDArray from scipy.sparse import ( @@ -1233,7 +1234,8 @@ class sparse_autocov_csr_mat: """ - def __init__(self, PT, S, symmetric=False): + def __init__(self, PT:csr_matrix, S:csr_matrix, + symmetric:bool=False): assert isspmatrix_csr(S) assert isspmatrix_csr(PT) @@ -1516,7 +1518,10 @@ class sparse_autocov_mat: """ - def __init__(self, PT, p1, p2, PT_symmetric=False): + def __init__(self, PT:csr_matrix, + p1: Union[float, int, np.number, NDArray], + p2: Union[float, int, np.number, NDArray], + PT_symmetric:bool=False): assert isspmatrix_csr(PT) if isinstance(p1, np.ndarray) and isinstance(p2, np.ndarray): From 55385fb74f23554a408b984313f0302b11fb722a Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Sep 2024 19:36:41 +0200 Subject: [PATCH 59/86] adding tests for sparse_autocov_mat inits --- tests/test_sparse_stoch_matrix.py | 32 +++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index f4628db..c7e22b9 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -1,3 +1,4 @@ +import pytest import numpy as np import tracemalloc @@ -39,6 +40,7 @@ def test_SSM_small(get_csr_matrix_small): # ### # inti from scipy.sparse.csr_matrix A_csr = get_csr_matrix_small + print(f"{A_csr[1,2]=}") ssm = SSM.from_full_csr_matrix(A_csr) np.testing.assert_equal(A_csr.toarray(), ssm.toarray(), strict=False) # crete a diagonal matrix @@ -323,15 +325,41 @@ def test_inplace_diag_matmul_csr(cs_matrix_creator): np.testing.assert_equal(A_csc.toarray(), Acsc_rescaled_row) +# ### +# Testing the autocovaraince matrix class +# ### def test_SparseAutocovMatrixCSR(): """Check basic operations on sparse_autocov_csr_mat""" from flowstab.SparseStochMat import sparse_autocov_csr_mat as SAMCSR + # ignoring this for now pass -def test_SparseAutocovMatrix(): +@pytest.mark.parametrize("p1, p2, size", + [(np.random.random(size=100000), + np.random.random(size=100000), 100000), + (0.2, 0.3, 100000), + (0.4, None, 100000), + (None, 0.1, 100000), + (None, None, 100000)]) +def test_SparseAutocovMatrix(p1, p2, size, cs_matrix_creator): """Check basic operations on sparse_autocov_mat""" from flowstab.SparseStochMat import sparse_autocov_mat as SAM - pass + from flowstab.SparseStochMat import ( + inplace_diag_matmul_csr + ) + T = cs_matrix_creator(nbr=1, size=size, nbr_non_zeros=1000)[0] + PT = T.copy() + if p1 is not None: + if not isinstance(p1, np.ndarray): + _p1 = np.full(shape=size, fill_value=p1) + else: + _p1 = p1 + inplace_diag_matmul_csr(PT, _p1) + # testing various init methods + np.testing.assert_equal( + SAM(PT=PT, p1=p1, p2=p2).PT.data, + SAM.from_T(T=T, p1=p1, p2=p2).PT.data + ) def test_sparse_matmul_mkl_memory(csr_matrix_creator): """ From a2d986dade864f2c42932ea12f108ea63d3e393f Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 17 Sep 2024 13:33:20 +0200 Subject: [PATCH 60/86] adding tests for SAM creation (also from T) --- tests/test_sparse_stoch_matrix.py | 55 +++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index c7e22b9..9409626 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -341,7 +341,7 @@ def test_SparseAutocovMatrixCSR(): (0.4, None, 100000), (None, 0.1, 100000), (None, None, 100000)]) -def test_SparseAutocovMatrix(p1, p2, size, cs_matrix_creator): +def test_SAM_init(p1, p2, size, cs_matrix_creator): """Check basic operations on sparse_autocov_mat""" from flowstab.SparseStochMat import sparse_autocov_mat as SAM from flowstab.SparseStochMat import ( @@ -355,12 +355,63 @@ def test_SparseAutocovMatrix(p1, p2, size, cs_matrix_creator): else: _p1 = p1 inplace_diag_matmul_csr(PT, _p1) - # testing various init methods + # testing various init methods with from_T + sam = SAM(PT=PT, p1=p1, p2=p2) + +@pytest.mark.parametrize("p1, p2, size", + [(np.random.random(size=100000), + np.random.random(size=100000), 100000), + (0.2, 0.3, 100000), + (0.4, None, 100000), + (None, 0.1, 100000), + (None, None, 100000)]) +def test_SAM_from_T(p1, p2, size, cs_matrix_creator): + """Check basic operations on sparse_autocov_mat""" + from flowstab.SparseStochMat import sparse_autocov_mat as SAM + from flowstab.SparseStochMat import ( + inplace_diag_matmul_csr + ) + T = cs_matrix_creator(nbr=1, size=size, nbr_non_zeros=1000)[0] + PT = T.copy() + if p1 is not None: + if not isinstance(p1, np.ndarray): + _p1 = np.full(shape=size, fill_value=p1) + else: + _p1 = p1 + inplace_diag_matmul_csr(PT, _p1) + # testing various init methods with from_T np.testing.assert_equal( SAM(PT=PT, p1=p1, p2=p2).PT.data, SAM.from_T(T=T, p1=p1, p2=p2).PT.data ) +@pytest.mark.parametrize("p1, p2, size", + [(np.random.random(size=100000), + np.random.random(size=100000), 100000), + (0.2, 0.3, 100000), + (0.4, None, 100000), + (None, 0.1, 100000), + (None, None, 100000)]) +def test_SAM_from_T_forward(p1, p2, size, cs_matrix_creator): + """Check basic operations on sparse_autocov_mat""" + from flowstab.SparseStochMat import sparse_autocov_mat as SAM + from flowstab.SparseStochMat import ( + inplace_diag_matmul_csr + ) + T = cs_matrix_creator(nbr=1, size=size, nbr_non_zeros=1000)[0] + PT = T.copy() + if p1 is not None: + if not isinstance(p1, np.ndarray): + _p1 = np.full(shape=size, fill_value=p1) + else: + _p1 = p1 + inplace_diag_matmul_csr(PT, _p1) + # testing various init methods with from_T_forward + np.testing.assert_equal( + SAM(PT=PT, p1=p1, p2=p2).PT.data, + SAM.from_T_forward(T=T, p1=p1, p2=p2).PT.data + ) + def test_sparse_matmul_mkl_memory(csr_matrix_creator): """ """ From f2c7cd6ce224703c07892b64b54d7d9802883052 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 17 Sep 2024 13:38:10 +0200 Subject: [PATCH 61/86] better docstring formatting --- src/flowstab/SparseStochMat.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index ea270ce..0a67126 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -1567,7 +1567,11 @@ def __repr__(self): @classmethod def from_T(cls, T, p1=None, p2=None): """Generate autocovariance matrix from transition matrix T as - S = diag(p1) @ T - p1^T @ p2. + + .. math:: + :label: autocov-matrix + + S = diag(p1) @ T - p1^T @ p2. Parameters ---------- @@ -1608,9 +1612,16 @@ def from_T(cls, T, p1=None, p2=None): return cls(PT=PT, p1=p1, p2=p2) @classmethod - def from_T_forward(cls, T, p1=None, p2=None): + def from_T_forward(cls, T:csr_matrix, + p1:Union[None, NDArray]=None, + p2:Union[None, NDArray]=None): """Generate the forward autocovariance matrix from transition matrix T as - S = diag(p1) @ T @ diag(1/p2) @ T.T @ diag(p1) - p1.T @ p1. + + .. math:: + :label: forward-autocov-matrix + + S = diag(p1) @ T @ diag(1/p2) @ T.T @ diag(p1) - p1.T @ p1. + Parameters ---------- From 41adc5329bb7097e251ceed1a775269674c6c8b5 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Tue, 17 Sep 2024 13:47:04 +0200 Subject: [PATCH 62/86] drop the math mode --- src/flowstab/SparseStochMat.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index 0a67126..f070dd1 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -1568,9 +1568,6 @@ def __repr__(self): def from_T(cls, T, p1=None, p2=None): """Generate autocovariance matrix from transition matrix T as - .. math:: - :label: autocov-matrix - S = diag(p1) @ T - p1^T @ p2. Parameters @@ -1617,9 +1614,6 @@ def from_T_forward(cls, T:csr_matrix, p2:Union[None, NDArray]=None): """Generate the forward autocovariance matrix from transition matrix T as - .. math:: - :label: forward-autocov-matrix - S = diag(p1) @ T @ diag(1/p2) @ T.T @ diag(p1) - p1.T @ p1. From 1f140789b58ae7a047f1b6d9e1daca0021bb1e1d Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 23 Sep 2024 14:59:43 +0200 Subject: [PATCH 63/86] ditch passing by possition --- src/flowstab/SparseStochMat.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index f070dd1..4880a4f 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -1272,9 +1272,9 @@ def __repr__(self): def copy(self): - return self.__class__(self.PT.copy(), - self.S.copy(), - copy(self.PT_symmetric)) + return self.__class__(PT=self.PT.copy(), + S=self.S.copy(), + symmetric=self.PT_symmetric) def toarray(self): @@ -1438,9 +1438,9 @@ def aggregate(self, idx_list): - return self.__class__(newPT.tocsr(), - newS.tocsr(), - self.PT_symmetric) + return self.__class__(PT=newPT.tocsr(), + S=newS.tocsr(), + symmetric=self.PT_symmetric) def is_all_zeros(self): @@ -1671,10 +1671,10 @@ def from_T_forward(cls, T:csr_matrix, def copy(self): - return self.__class__(self.PT.copy(), - copy(self.p1), - copy(self.p2), - copy(self.PT_symmetric)) + return self.__class__(PT=self.PT.copy(), + p1=copy(self.p1), + p2=copy(self.p2), + PT_symmetric=self.PT_symmetric) def toarray(self): @@ -1682,7 +1682,7 @@ def toarray(self): return self.PT.toarray() - np.ones(self.shape)*self.p1p2 else: - return self.PT.toarray() - np.outer(self.p1,self.p2) + return self.PT.toarray() - np.outer(self.p1, self.p2) def get_submat_sum(self, row_idx, col_idx): @@ -1868,7 +1868,9 @@ def aggregate(self, idx_list): newp1 = newp1/newp1.sum() newp2 = newp2/newp2.sum() - return self.__class__(newPT.tocsr(), newp1, newp2, self.PT_symmetric) + return self.__class__(PT=newPT.tocsr(), + p1=newp1, p2=newp2, + PT_symmetric=self.PT_symmetric) def is_all_zeros(self): From efbfacbde4f6b96fa191127de4668b03726db8b4 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 23 Sep 2024 15:00:17 +0200 Subject: [PATCH 64/86] including more methods for SAM --- tests/test_sparse_stoch_matrix.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 9409626..4ce0aa1 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -335,12 +335,12 @@ def test_SparseAutocovMatrixCSR(): pass @pytest.mark.parametrize("p1, p2, size", - [(np.random.random(size=100000), - np.random.random(size=100000), 100000), - (0.2, 0.3, 100000), - (0.4, None, 100000), - (None, 0.1, 100000), - (None, None, 100000)]) + [(np.random.random(size=1000), + np.random.random(size=1000), 1000), + (0.2, 0.3, 1000), + (0.4, None, 1000), + (None, 0.1, 1000), + (None, None, 1000)]) def test_SAM_init(p1, p2, size, cs_matrix_creator): """Check basic operations on sparse_autocov_mat""" from flowstab.SparseStochMat import sparse_autocov_mat as SAM @@ -357,6 +357,8 @@ def test_SAM_init(p1, p2, size, cs_matrix_creator): inplace_diag_matmul_csr(PT, _p1) # testing various init methods with from_T sam = SAM(PT=PT, p1=p1, p2=p2) + sam_copy = sam.copy() + sam_array = sam.toarray() @pytest.mark.parametrize("p1, p2, size", [(np.random.random(size=100000), From bce7c392e9be169a2f904ff0bc0e664228479f94 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Wed, 25 Sep 2024 20:34:23 +0200 Subject: [PATCH 65/86] started with synthtempnetwork --- src/flowstab/SynthTempNetwork.py | 19 ++++++++++++++++++- tests/test_synth_temp_nw.py | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/test_synth_temp_nw.py diff --git a/src/flowstab/SynthTempNetwork.py b/src/flowstab/SynthTempNetwork.py index 35e752b..69807d5 100644 --- a/src/flowstab/SynthTempNetwork.py +++ b/src/flowstab/SynthTempNetwork.py @@ -18,6 +18,7 @@ """ +from typing import Union from queue import Empty, PriorityQueue @@ -35,7 +36,23 @@ class Distro: they can also be changed when calling `Distro.draw_val(loc,scale)` """ - def __init__(self, loc=0, scale=1, dist_type="exponential"): + def __init__(self, + loc:Union[float, int]=0.0, + scale:Union[float, int]=1, + dist_type:str="exponential"): + """ + + Parameters + ---------- + loc: + Location parameter value of a SciPy distribution object + scale: + Scale parameter value of a SciPy distribution object + dist_type: + The type of distribution that will be drawn from + + Currently, only `"exponential"` (the default) is supported + """ self.loc=loc self.scale=scale self.dist_type=dist_type diff --git a/tests/test_synth_temp_nw.py b/tests/test_synth_temp_nw.py new file mode 100644 index 0000000..da1d7e3 --- /dev/null +++ b/tests/test_synth_temp_nw.py @@ -0,0 +1,20 @@ +from flowstab.SynthTempNetwork import ( + Individual, + SynthTempNetwork +) + +def test_distro(): + from flowstab.SynthTempNetwork import ( + Distro, + ) + d = Distro(loc=0.0, scale=1.0) + _ = d.draw_val() + _ = d.draw_val(loc=0.5) + _ = d.draw_val(scale=0.5) + _ = d.draw_val(loc=0.5, scale=1.5) + +def test_individual(): + from flowstab.SynthTempNetwork import ( + Individual, + ) + From fd983708ee45509d694b0edb5086b3b886364ef4 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Fri, 27 Sep 2024 15:15:22 +0200 Subject: [PATCH 66/86] adding minimal tests for synth_temp_network subm --- tests/test_synth_temp_nw.py | 40 ++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/test_synth_temp_nw.py b/tests/test_synth_temp_nw.py index da1d7e3..354b51e 100644 --- a/tests/test_synth_temp_nw.py +++ b/tests/test_synth_temp_nw.py @@ -1,7 +1,4 @@ -from flowstab.SynthTempNetwork import ( - Individual, - SynthTempNetwork -) +import pytest def test_distro(): from flowstab.SynthTempNetwork import ( @@ -13,8 +10,41 @@ def test_distro(): _ = d.draw_val(scale=0.5) _ = d.draw_val(loc=0.5, scale=1.5) -def test_individual(): + +# you are here! +@pytest.mark.parametrize( + "_id, i_d_loc, i_d_scale, i_d_mf, a_d_loc, a_d_scale, a_d_mf, dist_type, group", + [(1, 0.0, 1.0, None, 0.0, 1.0, None, "experimental", 0),] + ) +def test_individual(_id, + i_d_loc, i_d_scale, i_d_mf, + a_d_loc, a_d_scale, a_d_mf, + dist_type, group): from flowstab.SynthTempNetwork import ( Individual, ) + i1 = Individual(ID=_id, + inter_distro_loc=i_d_loc, + inter_distro_scale=i_d_scale, + inter_distro_type=dist_type, + inter_distro_mod_func=i_d_mf, + activ_distro_loc=a_d_loc, + activ_distro_scale=a_d_scale, + activ_distro_type=dist_type, + activ_distro_mod_func=a_d_mf, + group=group) + _ = i1.draw_inter_duration(time=None) + _ = i1.draw_inter_duration(time=1.0) + _ = i1.draw_activ_time(time=None) + _ = i1.draw_activ_time(time=1.0) + +def test_synth_temp_network(): + from flowstab.SynthTempNetwork import ( + Individual, + SynthTempNetwork + ) + individuals = [Individual(i, group=0) for i in range(20)] + sim = SynthTempNetwork(individuals, t_start=0, t_end=50) + + sim.run(save_all_states=True, save_dt_states=True, verbose=True) From 2aedd354ede3a1d79c8e1d9bfc5e574a8f311a9a Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Fri, 27 Sep 2024 15:23:34 +0200 Subject: [PATCH 67/86] started with flow_stability sub-m --- src/flowstab/FlowStability.py | 23 +++++++++++++++++++---- tests/test_flow_stability.py | 1 + 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 tests/test_flow_stability.py diff --git a/src/flowstab/FlowStability.py b/src/flowstab/FlowStability.py index 3531a30..20cdfd1 100644 --- a/src/flowstab/FlowStability.py +++ b/src/flowstab/FlowStability.py @@ -18,6 +18,9 @@ """ +from __future__ import annotations +from typing import Collection + import importlib.util import os import time @@ -56,10 +59,22 @@ class Partition: and a node to cluster dict. """ - def __init__(self, num_nodes, - cluster_list=None, - node_to_cluster_dict=None, - check_integrity=False): + def __init__(self, + num_nodes:int, + cluster_list:Collection|None=None, + node_to_cluster_dict:dict|None=None, + check_integrity:bool=False): + """ + Parameters + ---------- + num_nodes: + The number of nodes in the partition + cluster_list: + A list of clusters with each cluster being a set of nodes + node_to_custer_dict: + A mapping that maps each node to the index of the corresponding + cluster in `cluster_list` + """ self.num_nodes = num_nodes diff --git a/tests/test_flow_stability.py b/tests/test_flow_stability.py new file mode 100644 index 0000000..5871ed8 --- /dev/null +++ b/tests/test_flow_stability.py @@ -0,0 +1 @@ +import pytest From 3af2c87f80b506b876465c0f71dc8f606a2f71fa Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 10:32:27 +0200 Subject: [PATCH 68/86] initiating FlowStability tests --- tests/test_flow_stability.py | 84 ++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/test_flow_stability.py b/tests/test_flow_stability.py index 5871ed8..c05fcd6 100644 --- a/tests/test_flow_stability.py +++ b/tests/test_flow_stability.py @@ -1 +1,85 @@ import pytest + +def test_Partition(): + """ + """ + from flowstab.FlowStability import Partition + pass + +def test_BaseClustering(): + """ + """ + from flowstab.FlowStability import BaseClustering + pass + +def test_Clustering(): + """ + """ + from flowstab.FlowStability import Clustering + pass + +def test_SparseClustering(): + """ + """ + from flowstab.FlowStability import SparseClustering + pass + +def test_FlowIntegralClustering(): + """ + """ + from flowstab.FlowStability import FlowIntegralClustering + pass + +def test_jaccard_distance(): + """ + """ + from flowstab.FlowStability import jaccard_distance + pass + +def test_norm_mutual_information(): + """ + """ + from flowstab.FlowStability import norm_mutual_information + pass + +def test_norm_var_information(): + """ + """ + from flowstab.FlowStability import norm_var_information + pass + +def test_norm_var_information(): + """ + """ + from flowstab.FlowStability import norm_var_information + pass + +def test_avg_norm_var_information(): + """ + """ + from flowstab.FlowStability import avg_norm_var_information + pass + +def test_static_clustering(): + """ + """ + from flowstab.FlowStability import static_clustering + pass + +def test_n_random_seeds(): + """ + """ + from flowstab.FlowStability import n_random_seeds + pass + +def test_run_multi_louvain(): + """ + """ + from flowstab.FlowStability import run_multi_louvain + pass + +def test_sort_clusters(): + """ + """ + from flowstab.FlowStability import sort_clusters + pass From 043c898d872d77497ce687795f76313fc6cc4a5b Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 10:42:51 +0200 Subject: [PATCH 69/86] init tests for parallel_expm --- tests/test_parallel_expm.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/test_parallel_expm.py diff --git a/tests/test_parallel_expm.py b/tests/test_parallel_expm.py new file mode 100644 index 0000000..380ea5f --- /dev/null +++ b/tests/test_parallel_expm.py @@ -0,0 +1,24 @@ +import pytest + +import numpy as np + +from flowstab.parallel_clustering import ( + # _init_worker, + # _worker, + compute_parallel_expm, + # _stack_sparse_cols, + # _expm_worke, + compute_subspace_expm_parallel +) + +def test_compute_parallel_expm(): + """ + """ + from flowstab.parallel_clustering import compute_parallel_expm + pass + +def test_compute_subspace_parallel_expm(): + """ + """ + from flowstab.parallel_clustering import compute_subspace_expm_parallel + pass From db1fecc4809395dee045d8d1a8d20f2d2f2b25d8 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 11:14:19 +0200 Subject: [PATCH 70/86] init tests for temporal_network --- tests/test_temporal_network.py | 85 ++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/test_temporal_network.py diff --git a/tests/test_temporal_network.py b/tests/test_temporal_network.py new file mode 100644 index 0000000..ab2b5c0 --- /dev/null +++ b/tests/test_temporal_network.py @@ -0,0 +1,85 @@ +import pytest + +def test_ContTempNetwork(): + """ + """ + from flowstab.TemporalNetwork import ContTempNetwork + pass + +def test_ContTempInstNetwork(): + """ + """ + from flowstab.TemporalNetwork import ContTempInstNetwork + pass + +def test_lin_approx_trans_matrix(): + """ + """ + from flowstab.TemporalNetwork import lin_approx_trans_matrix + pass + +def test_compute_stationary_transition(): + """ + """ + from flowstab.TemporalNetwork import compute_stationary_transition + pass + +def test_compute_subspace_expm(): + """ + """ + from flowstab.TemporalNetwork import compute_subspace_expm + pass + +def test_csc_row_normalize(): + """ + """ + from flowstab.TemporalNetwork import csc_row_normalize + pass + +def test_find_spectral_gap(): + """ + """ + from flowstab.TemporalNetwork import find_spectral_gap + pass + +def test_remove_nnz_rowcol(): + """ + """ + from flowstab.TemporalNetwork import remove_empty_clusters + pass + +def test_rebuild_nnz_rowcol(): + """ + """ + from flowstab.TemporalNetwork import numpy_rebuild_nnz_rowcol + pass + +def test_sparse_lapl_expm(): + """ + """ + from flowstab.TemporalNetwork import sparse_lapl_expm + pass + +def test_sparse_lin_approx(): + """ + """ + from flowstab.TemporalNetwork import sparse_lin_approx + pass + +def test_sparse_stationary_trans(): + """ + """ + from flowstab.TemporalNetwork import sparse_stationary_trans + pass + +def test_set_to_ones(): + """ + """ + from flowstab.TemporalNetwork import set_to_ones + pass + +def test_set_to_zeroes(): + """ + """ + from flowstab.TemporalNetwork import set_to_zeroes + pass From decc937780bad170a2fc2d9050bdbe80ce1a0cf0 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 11:25:19 +0200 Subject: [PATCH 71/86] init tests for parallel_clustering --- tests/test_parallel_clustering.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/test_parallel_clustering.py diff --git a/tests/test_parallel_clustering.py b/tests/test_parallel_clustering.py new file mode 100644 index 0000000..f301a5f --- /dev/null +++ b/tests/test_parallel_clustering.py @@ -0,0 +1,19 @@ +import pytest + +def test_compute_parallel_clustering(): + """ + """ + from flowstab.parallel_clustering import compute_parallel_clustering + pass + +def test_n_random_seeds(): + """ + """ + from flowstab.parallel_clustering import n_random_seeds + pass + +def test_compute_parallel_nvi(): + """ + """ + from flowstab.parallel_clustering import compute_parallel_nvi + pass From 6076d986e94920b31adcd823dc1db9438a78b931 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 12:00:57 +0200 Subject: [PATCH 72/86] minor formatting --- src/flowstab/FlowStability.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/flowstab/FlowStability.py b/src/flowstab/FlowStability.py index 20cdfd1..3dd1bab 100644 --- a/src/flowstab/FlowStability.py +++ b/src/flowstab/FlowStability.py @@ -43,11 +43,21 @@ sparse_matmul, sparse_stoch_mat, ) -from .TemporalNetwork import inplace_csr_row_normalize, set_to_zeroes, sparse_lapl_expm +from .TemporalNetwork import ( + inplace_csr_row_normalize, + set_to_zeroes, + sparse_lapl_expm +) USE_CYTHON = True if importlib.util.find_spec("cython") is not None: - from _cython_fast_funcs import compute_S, cython_nmi, cython_nvi, sum_Sout, sum_Sto + from _cython_fast_funcs import ( + compute_S, + cython_nmi, + cython_nvi, + sum_Sout, + sum_Sto + ) else: print("Could not load cython functions") USE_CYTHON = False @@ -2378,6 +2388,7 @@ def static_clustering(A, t=1, rnd_seed=None, discrete_time_rw=False, return Clustering(p1=pi,p2=pi,T=T, rnd_seed=rnd_seed) +# TODO: move to helper scripts def n_random_seeds(n): # generate n random seeds From aa071ce14dc1f5fdbb5ed9ae63e4e7ba49953cf4 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 12:02:27 +0200 Subject: [PATCH 73/86] minor notes/info --- src/flowstab/_cython_fast_funcs.pyx | 4 ++++ src/flowstab/scripts/run_clusterings.py | 1 + 2 files changed, 5 insertions(+) diff --git a/src/flowstab/_cython_fast_funcs.pyx b/src/flowstab/_cython_fast_funcs.pyx index 5fcdd9b..3f3c5d4 100644 --- a/src/flowstab/_cython_fast_funcs.pyx +++ b/src/flowstab/_cython_fast_funcs.pyx @@ -69,6 +69,10 @@ def sum_Sout(double[:, ::1] S , int k, list ix_ci): @cython.boundscheck(False) # Deactivate bounds checking @cython.wraparound(False) # Deactivate negative indexing def compute_S(double[:] p1, double[:] p2, double[:,:] T): + """Computes the internal matrix comparing probabilities for each node + + S[i,j] = p1[i]*T[i,j] - p1[i]*p2[j] + """ cdef Py_ssize_t imax = T.shape[0] cdef Py_ssize_t jmax = T.shape[1] diff --git a/src/flowstab/scripts/run_clusterings.py b/src/flowstab/scripts/run_clusterings.py index ff9b3ef..d4698f7 100644 --- a/src/flowstab/scripts/run_clusterings.py +++ b/src/flowstab/scripts/run_clusterings.py @@ -619,6 +619,7 @@ def compute_static_clustering_fct(params): return (stat_clustering.partition.cluster_list, stat_clustering.compute_stability(), seed) +# TODO: move to helper scripts def n_random_seeds(n): return [int.from_bytes(os.urandom(4), byteorder="big") for \ From 762faf971f3b9df2592d5fc314ff3699dc655c1f Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Sun, 29 Sep 2024 14:03:12 +0200 Subject: [PATCH 74/86] minor edits; init cython testing --- tests/conftest.py | 17 +++++++++++++++++ tests/test_cython_fast_funcs.py | 14 ++++++++++++++ tests/test_parallel_expm.py | 5 ++--- 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 tests/test_cython_fast_funcs.py diff --git a/tests/conftest.py b/tests/conftest.py index 694bab2..d7f5c7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -87,3 +87,20 @@ def compare_sparse_matrice(A, B): np.testing.assert_equal(A.indices[A_s:A_e][A_sorted], B.indices[B_s:B_e][B_sorted]) np.testing.assert_equal(B.data[B_s:B_e][B_sorted], A.data[A_s:A_e][A_sorted]) return compare_sparse_matrice + +@pytest.fixture(scope='session') +def probabilities_transition(): + """Create exemplary densities and transition probabilities + """ + nbr_non_zeros = 1000 + p1 = np.ones(shape=(nbr_non_zeros), dtype=np.float64) / nbr_non_zeros + T = np.zeros(shape=(nbr_non_zeros, nbr_non_zeros), dtype=np.float64) + for i in range(nbr_non_zeros): + if np.random.rand() >= 0.5: + _t = np.random.dirichlet(np.ones(nbr_non_zeros),size=1) + else: + _t = np.zeros(shape=(nbr_non_zeros,), dtype=np.float64) + T[:,i] = _t + + p2 = p1 @ T + return p1, p2, T diff --git a/tests/test_cython_fast_funcs.py b/tests/test_cython_fast_funcs.py new file mode 100644 index 0000000..c4fcd2d --- /dev/null +++ b/tests/test_cython_fast_funcs.py @@ -0,0 +1,14 @@ +import pytest + +import numpy as np + +def test_compute_S(probabilities_transition): + """ + """ + from _cython_fast_funcs import ( + compute_S + ) + p1, p2, T = probabilities_transition + S = compute_S(p1=p1, p2=p2, T=T) + S_nonC = np.diag(p1) @ T - np.outer(p1, p2) + np.testing.assert_equal(S, S_nonC) diff --git a/tests/test_parallel_expm.py b/tests/test_parallel_expm.py index 380ea5f..1e30c31 100644 --- a/tests/test_parallel_expm.py +++ b/tests/test_parallel_expm.py @@ -5,10 +5,9 @@ from flowstab.parallel_clustering import ( # _init_worker, # _worker, - compute_parallel_expm, + compute_parallel_clustering, # _stack_sparse_cols, # _expm_worke, - compute_subspace_expm_parallel ) def test_compute_parallel_expm(): @@ -17,7 +16,7 @@ def test_compute_parallel_expm(): from flowstab.parallel_clustering import compute_parallel_expm pass -def test_compute_subspace_parallel_expm(): +def test_compute_subspace_expm_parallel(): """ """ from flowstab.parallel_clustering import compute_subspace_expm_parallel From 0f9be607c67e90731bfb19d2d9c2486481f89eed Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 11:55:47 +0200 Subject: [PATCH 75/86] simple test cases --- tests/test_cython_fast_funcs.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_cython_fast_funcs.py b/tests/test_cython_fast_funcs.py index c4fcd2d..8eb99dd 100644 --- a/tests/test_cython_fast_funcs.py +++ b/tests/test_cython_fast_funcs.py @@ -2,6 +2,40 @@ import numpy as np +def test_sum_Sto(): + """ + """ + from _cython_fast_funcs import ( + sum_Sto + ) + # Define a sample S array and ix_cf list for testing purposes. + S = np.array([[1,2,3], [4,5,6], [7,8,9]], dtype=np.float64) + k = 0 + ix_cf = [1, 2] + # Calculate the expected result manually. + expected_result = S[k, ix_cf[0]] + S[ix_cf[0], k] + S[k, ix_cf[1]] + \ + S[ix_cf[1], k] + S[k,k] + # Compare the expected result to the actual output from sum_Sto. + assert np.testing.allclose(expected_result,sum_Sto(S, k, ix_cf)) + +def test_sum_Sout(): + """ + """ + from _cython_fast_funcs import ( + sum_Sout + ) + # Define a sample S array and ix_ci list for testing purposes. + S = np.array([[1,2,3], [4,5,6], [7,8,9]], dtype=np.float64) + k = 0 + ix_ci = [1, 2] + + # Calculate the expected result manually. + expected_result = -S[k, ix_ci[0]] - S[ix_ci[0], k] - S[k, ix_ci[1]] - \ + S[ix_ci[1], k] + S[k,k] + + # Compare the expected result to the actual output from sum_Sout. + assert np.testing.allclose(expected_result,sum_Sout(S, k, ix_ci)) + def test_compute_S(probabilities_transition): """ """ From 1c3f36b924f1555121d54db0aa133c4627f1ecad Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 12:10:40 +0200 Subject: [PATCH 76/86] better bassis for test data generation --- tests/conftest.py | 25 +++++++++++++++++++++++++ tests/test_cython_fast_funcs.py | 6 ++---- tests/test_parallel_expm.py | 14 ++------------ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d7f5c7a..3c89e2e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -104,3 +104,28 @@ def probabilities_transition(): p2 = p1 @ T return p1, p2, T + +@pytest.fixture(scope='session') +def propa_transproba_creator(): + """Creat an exemplary csr matrix that can be used for testing + """ + size = 1000 + zero_col_density = 0.05 + def _get_p_tp(nbr:int=1, size:int=size, + zero_col_density:float=zero_col_density): + """Generates a tuple of p1,p2,T triplets for a network of `size` nodes + """ + ptps = [] + for _ in range(nbr): + p1 = np.ones(shape=(size), dtype=np.float64) / size + T = np.zeros(shape=(size, size), dtype=np.float64) + for i in range(size): + if np.random.rand() < 1 - zero_col_density: + _t = np.random.dirichlet(np.ones(size),size=1) + else: + _t = np.zeros(shape=(size,), dtype=np.float64) + T[:,i] = _t + p2 = p1 @ T + ptps.append((p1,p2,T)) + return tuple(ptps) + return _get_p_tp diff --git a/tests/test_cython_fast_funcs.py b/tests/test_cython_fast_funcs.py index 8eb99dd..cacff37 100644 --- a/tests/test_cython_fast_funcs.py +++ b/tests/test_cython_fast_funcs.py @@ -28,21 +28,19 @@ def test_sum_Sout(): S = np.array([[1,2,3], [4,5,6], [7,8,9]], dtype=np.float64) k = 0 ix_ci = [1, 2] - # Calculate the expected result manually. expected_result = -S[k, ix_ci[0]] - S[ix_ci[0], k] - S[k, ix_ci[1]] - \ S[ix_ci[1], k] + S[k,k] - # Compare the expected result to the actual output from sum_Sout. assert np.testing.allclose(expected_result,sum_Sout(S, k, ix_ci)) -def test_compute_S(probabilities_transition): +def test_compute_S(propa_transproba_creator): """ """ from _cython_fast_funcs import ( compute_S ) - p1, p2, T = probabilities_transition + p1, p2, T = propa_transproba_creator(nbr=1,size=1000)[0] S = compute_S(p1=p1, p2=p2, T=T) S_nonC = np.diag(p1) @ T - np.outer(p1, p2) np.testing.assert_equal(S, S_nonC) diff --git a/tests/test_parallel_expm.py b/tests/test_parallel_expm.py index 1e30c31..3e3311d 100644 --- a/tests/test_parallel_expm.py +++ b/tests/test_parallel_expm.py @@ -1,23 +1,13 @@ import pytest -import numpy as np - -from flowstab.parallel_clustering import ( - # _init_worker, - # _worker, - compute_parallel_clustering, - # _stack_sparse_cols, - # _expm_worke, -) - def test_compute_parallel_expm(): """ """ - from flowstab.parallel_clustering import compute_parallel_expm + from flowstab.parallel_expm import compute_parallel_expm pass def test_compute_subspace_expm_parallel(): """ """ - from flowstab.parallel_clustering import compute_subspace_expm_parallel + from flowstab.parallel_expm import compute_subspace_expm_parallel pass From f64641cc84eb8b624384c724bcb0f926ff9c3339 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 13:54:43 +0200 Subject: [PATCH 77/86] tests for temp networks --- tests/test_temporal_network.py | 152 +++++++++++++++++++++++++++++++-- 1 file changed, 146 insertions(+), 6 deletions(-) diff --git a/tests/test_temporal_network.py b/tests/test_temporal_network.py index ab2b5c0..db1e289 100644 --- a/tests/test_temporal_network.py +++ b/tests/test_temporal_network.py @@ -1,10 +1,150 @@ import pytest -def test_ContTempNetwork(): - """ - """ - from flowstab.TemporalNetwork import ContTempNetwork - pass +import os +import pickle +import tempfile + +import numpy as np +import pandas as pd + + +from flowstab.TemporalNetwork import ContTempNetwork + +class TestContTempNetwork: + def setup_method(self): + self.source_nodes = [0, 1] + self.target_nodes = [1, 2] + self.starting_times = [0.5, 1.0] + self.ending_times = [1.0, 1.5] + + self.extra_attrs = {"attr1": [True, False]} + self.events_table = pd.DataFrame({ + "source_nodes": self.source_nodes, + "target_nodes": self.target_nodes, + "starting_times": self.starting_times, + "ending_times": self.ending_times + }) + self.temp_dir = tempfile.gettempdir() + self.tmp_pkl = tempfile.NamedTemporaryFile(suffix='.pkl', delete=False) + self.tmp_json = tempfile.NamedTemporaryFile(suffix='.json', + delete=False) + + def test_init_with_events_table(self): + network = ContTempNetwork(events_table=self.events_table) + assert network.events_table.equals(self.events_table) + + def test_init_with_source_and_target_nodes(self): + network = ContTempNetwork(source_nodes=self.source_nodes, + target_nodes=self.target_nodes, + starting_times=self.starting_times, + ending_times=self.ending_times) + assert isinstance(network, ContTempNetwork) + + def test_init_without_source_nodes(self): + with pytest.raises(AssertionError): + ContTempNetwork(target_nodes=[1, 2]) + + def test_init_without_target_nodes(self): + with pytest.raises(AssertionError): + ContTempNetwork(source_nodes=[0, 1]) + + def test_init_with_invalid_source_node_type(self): + with pytest.raises(AssertionError): + ContTempNetwork(source_nodes=['a', 'b'], target_nodes=[1, 2]) + + def test_init_with_invalid_target_node_type(self): + with pytest.raises(AssertionError): + ContTempNetwork(source_nodes=[0, 1], target_nodes=['a', 'b']) + + def test_relabel_nodes(self): + # create a network without relabeling nodes + network = ContTempNetwork(source_nodes=[1, 2], target_nodes=[2, 3], + starting_times=self.starting_times, + ending_times=self.ending_times) + original_labels = network.node_to_label_dict + + # check that the node labels have been correctly converted to integers + new_network = ContTempNetwork(events_table=network.events_table, + relabel_nodes=False) + assert original_labels == {l: n + for l, n in enumerate(sorted([1, 2, 3]))} + # The new network should not have the correct ids + np.testing.assert_equal(new_network.node_array,np.array([0,1,2])) + + @pytest.fixture + def saved_network(self): + network = ContTempNetwork(source_nodes=self.source_nodes, + target_nodes=self.target_nodes, + starting_times=self.starting_times, + ending_times=self.ending_times) + with open(self.tmp_pkl.name, 'wb') as f: + pickle.dump(network, f) + return network + + @pytest.fixture + def get_loaded_network(self): + def loaded_network(): + with open(self.tmp_pkl.name, 'rb') as f: + return pickle.load(f) + return loaded_network + + def test_save_and_load_pickle(self, saved_network, get_loaded_network): + assert isinstance(saved_network, ContTempNetwork) + loaded_network = get_loaded_network() + assert isinstance(loaded_network, ContTempNetwork) + sn_et = saved_network.events_table + ln_et = loaded_network.events_table + pd.testing.assert_series_equal(sn_et.source_nodes, ln_et.source_nodes) + pd.testing.assert_series_equal(sn_et.target_nodes, ln_et.target_nodes) + pd.testing.assert_series_equal(sn_et.starting_times, ln_et.starting_times) + pd.testing.assert_series_equal(sn_et.ending_times, ln_et.ending_times) + + def test_relabel_nodes_with_custom_labels(self): + custom_labels = {"node0": 3, "node1": 4} + network = ContTempNetwork(source_nodes=["node0", "node1"], + target_nodes=["node1", "node2"], + starting_times=self.starting_times, + ending_times=self.ending_times, + node_to_label_dict=custom_labels) + new_network = ContTempNetwork(events_table=network.events_table) + nw_node_labels = set(new_network.node_to_label_dict.items()) + custom_labels = set(new_network.node_to_label_dict.items()) + assert all(nl in nw_node_labels for nl in custom_labels) + + def test_merge_overlapping_events(self): + # create a network with overlapping events + source_nodes = [0, 0] + target_nodes = [1, 2] + starting_times = [0.5, 1.0] + ending_times = [1.0, 1.5] + extra_attrs = {"attr1": [True, False]} + events_table = pd.DataFrame({ + "source_nodes": source_nodes, + "target_nodes": target_nodes, + "starting_times": starting_times, + "ending_times": ending_times + }) + network = ContTempNetwork(events_table=events_table, + merge_overlapping_events=True) + assert network._overlapping_events_merged + + def test_compute_times(self): + # create a network and check that the compute times dictionary is empty + network = ContTempNetwork() + assert not network._compute_times + + def teardown_method(self): + temp_dir = tempfile.gettempdir() + for file in os.listdir(temp_dir): + if 'temp.pkl' in file or 'temp.json' in file: + os.remove(os.path.join(temp_dir, file)) + +def test_ContTempNetworkErrors(): + with pytest.raises(AssertionError): + ContTempNetwork(source_nodes=[0, 1], target_nodes=[1]) + + with pytest.raises(Exception): + ContTempNetwork(events_table=pd.DataFrame({"source_nodes": [0, 1]})) def test_ContTempInstNetwork(): """ @@ -45,7 +185,7 @@ def test_find_spectral_gap(): def test_remove_nnz_rowcol(): """ """ - from flowstab.TemporalNetwork import remove_empty_clusters + from flowstab.TemporalNetwork import remove_nnz_rowcol pass def test_rebuild_nnz_rowcol(): From f08d01fe3bc68bedf03efb7fd10f3c0b9d8e12a9 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 13:55:05 +0200 Subject: [PATCH 78/86] minor fixes --- tests/test_cython_fast_funcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cython_fast_funcs.py b/tests/test_cython_fast_funcs.py index cacff37..ec8319c 100644 --- a/tests/test_cython_fast_funcs.py +++ b/tests/test_cython_fast_funcs.py @@ -16,7 +16,7 @@ def test_sum_Sto(): expected_result = S[k, ix_cf[0]] + S[ix_cf[0], k] + S[k, ix_cf[1]] + \ S[ix_cf[1], k] + S[k,k] # Compare the expected result to the actual output from sum_Sto. - assert np.testing.allclose(expected_result,sum_Sto(S, k, ix_cf)) + assert np.testing.assert_allclose(expected_result,sum_Sto(S, k, ix_cf)) def test_sum_Sout(): """ @@ -32,7 +32,7 @@ def test_sum_Sout(): expected_result = -S[k, ix_ci[0]] - S[ix_ci[0], k] - S[k, ix_ci[1]] - \ S[ix_ci[1], k] + S[k,k] # Compare the expected result to the actual output from sum_Sout. - assert np.testing.allclose(expected_result,sum_Sout(S, k, ix_ci)) + assert np.testing.assert_allclose(expected_result,sum_Sout(S, k, ix_ci)) def test_compute_S(propa_transproba_creator): """ From 392432c040470ab5605d7e0ca8c3b4feb64c6006 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 15:06:42 +0200 Subject: [PATCH 79/86] scipy upgrade - use sparse_dot_mk for mkl support --- requirements/base.txt | 2 +- requirements/mkl.txt | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 483f38b..e48f191 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,4 +2,4 @@ Cython>=3.0.11 pandas>=2.2.2 hickle>=5.0.3 psutil>=6.0.0 -scipy>=1.10 +scipy>=1.13 diff --git a/requirements/mkl.txt b/requirements/mkl.txt index e7a6c70..56cd4ba 100644 --- a/requirements/mkl.txt +++ b/requirements/mkl.txt @@ -1,3 +1 @@ -intel-scipy -# -i https://pypi.anaconda.org/intel/simple -# scipy>=1.10 +sparse_dot_mkl>=0.9.4 From 2a8f2018d65d6bed2e2305bbab9ec545949fb96c Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 15:07:47 +0200 Subject: [PATCH 80/86] minor formatting --- src/flowstab/TemporalNetwork.py | 3 --- tests/test_temporal_network.py | 14 ++++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/flowstab/TemporalNetwork.py b/src/flowstab/TemporalNetwork.py index e5439aa..897d225 100644 --- a/src/flowstab/TemporalNetwork.py +++ b/src/flowstab/TemporalNetwork.py @@ -1871,9 +1871,6 @@ def _compute_delta_trans_mat(self, lamda, round_zeros=True, tol=1e-8): print("PID ", os.getpid(), " : ", "delta_inter_T_lin has not been computed") - - - class ContTempInstNetwork(ContTempNetwork): """Continuous time temporal network with instantaneous events diff --git a/tests/test_temporal_network.py b/tests/test_temporal_network.py index db1e289..9a6fe85 100644 --- a/tests/test_temporal_network.py +++ b/tests/test_temporal_network.py @@ -7,6 +7,8 @@ import numpy as np import pandas as pd +from scipy.sparse import csr_matrix + from flowstab.TemporalNetwork import ContTempNetwork @@ -29,6 +31,12 @@ def setup_method(self): self.tmp_json = tempfile.NamedTemporaryFile(suffix='.json', delete=False) + def teardown_method(self): + temp_dir = tempfile.gettempdir() + for file in os.listdir(temp_dir): + if 'temp.pkl' in file or 'temp.json' in file: + os.remove(os.path.join(temp_dir, file)) + def test_init_with_events_table(self): network = ContTempNetwork(events_table=self.events_table) assert network.events_table.equals(self.events_table) @@ -133,12 +141,6 @@ def test_compute_times(self): network = ContTempNetwork() assert not network._compute_times - def teardown_method(self): - temp_dir = tempfile.gettempdir() - for file in os.listdir(temp_dir): - if 'temp.pkl' in file or 'temp.json' in file: - os.remove(os.path.join(temp_dir, file)) - def test_ContTempNetworkErrors(): with pytest.raises(AssertionError): ContTempNetwork(source_nodes=[0, 1], target_nodes=[1]) From 98d8fd8227afaa94d1293153f4f7bbcab48f5f79 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 30 Sep 2024 18:41:44 +0200 Subject: [PATCH 81/86] minor fix in assertion --- tests/test_cython_fast_funcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cython_fast_funcs.py b/tests/test_cython_fast_funcs.py index ec8319c..474045f 100644 --- a/tests/test_cython_fast_funcs.py +++ b/tests/test_cython_fast_funcs.py @@ -16,7 +16,7 @@ def test_sum_Sto(): expected_result = S[k, ix_cf[0]] + S[ix_cf[0], k] + S[k, ix_cf[1]] + \ S[ix_cf[1], k] + S[k,k] # Compare the expected result to the actual output from sum_Sto. - assert np.testing.assert_allclose(expected_result,sum_Sto(S, k, ix_cf)) + np.testing.assert_allclose(expected_result,sum_Sto(S, k, ix_cf)) def test_sum_Sout(): """ @@ -32,7 +32,7 @@ def test_sum_Sout(): expected_result = -S[k, ix_ci[0]] - S[ix_ci[0], k] - S[k, ix_ci[1]] - \ S[ix_ci[1], k] + S[k,k] # Compare the expected result to the actual output from sum_Sout. - assert np.testing.assert_allclose(expected_result,sum_Sout(S, k, ix_ci)) + np.testing.assert_allclose(expected_result,sum_Sout(S, k, ix_ci)) def test_compute_S(propa_transproba_creator): """ From 57ca6c860a0802966e26804675e0c8cc6027ad8d Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Dec 2024 23:00:49 +0100 Subject: [PATCH 82/86] cleaning up old tests --- tests/test_sparse_stoch_matrix.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 4ce0aa1..aa96acf 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -9,7 +9,6 @@ ) from flowstab.SparseStochMat import ( - csr_add, csr_matmul, csr_csc_matmul, csr_csrT_matmul, @@ -195,23 +194,6 @@ def test_rebuild_nnz_rowcol(cs_matrix_creator, compare_alike): # ### Testing the csr operations # ### -def test_csr_add_compare(cs_matrix_creator, compare_alike): - """Compare the csr_add to a native implementation""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000, - nbr_non_zeros=500, mode='r') - sum_csr_native = A_ssm1 + A_ssm2 - sum_csr = csr_add(A_ssm1, A_ssm2) - # np.testing.assert_allclose(sum_csr_native.data, sum_csr.data) - # np.testing.assert_equal(sum_csr_native.indices, sum_csr.indices) - # np.testing.assert_equal(sum_csr_native.indptr, sum_csr.indptr) - compare_alike(sum_csr, sum_csr_native) - -def test_csr_add_memory(cs_matrix_creator): - """Check the csr_add function for timing and memory consumption""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=1000000, - nbr_non_zeros=20000, mode='r') - _ = csr_add(A_ssm1, A_ssm2) - def test_csr_add_native_memory(cs_matrix_creator): """Check the csr native addition for timing and memory consumption""" A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=1000000, From 835700f143956e7f6a4d5e3bc88ef62cab758d46 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Dec 2024 23:12:34 +0100 Subject: [PATCH 83/86] cleaning up all tests for discarded implementations --- tests/test_sparse_stoch_matrix.py | 68 ------------------------------- 1 file changed, 68 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index aa96acf..5336e36 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -8,13 +8,6 @@ eye, ) -from flowstab.SparseStochMat import ( - csr_matmul, - csr_csc_matmul, - csr_csrT_matmul, -) - - def test_timing(capfd): """Bacic operations with the 'spares_stoch_mat' class """ @@ -200,79 +193,18 @@ def test_csr_add_native_memory(cs_matrix_creator): nbr_non_zeros=20000, mode='r') _ = A_ssm1 + A_ssm2 -def test_csr_matmul_compare(cs_matrix_creator, compare_alike): - """Compare the csr_matmul to a native implementation""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=100000, - nbr_non_zeros=2000, mode='r') - mmul_csr_native = A_ssm1 @ A_ssm2 - mmul_csr = csr_matmul(A_ssm1, A_ssm2) - # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) - # np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) - # np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) - compare_alike(mmul_csr_native, mmul_csr) - -def test_csr_matmul_native_memory(cs_matrix_creator): - """Check the csr_matmul function for timing and memory consumption""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000000, - nbr_non_zeros=200000, mode='r') - _ = A_ssm1 @ A_ssm2 - -def test_csr_matmul_memory(cs_matrix_creator): - """Check the csr native @ function for timing and memory consumption""" - A_ssm1, A_ssm2 = cs_matrix_creator(nbr=2, size=10000000, - nbr_non_zeros=200000, mode='r') - _ = csr_matmul(A_ssm1, A_ssm2) - -def test_csr_csc_matmul_compare(cs_matrix_creator, compare_alike): - """Compare the csr_csc_matmul to a native implementation""" - A_csr, = cs_matrix_creator(nbr=1, size=100000, - nbr_non_zeros=2000, mode='r') - A_csc, = cs_matrix_creator(nbr=1, size=100000, - nbr_non_zeros=2000, mode='c') - mmul_csr_native = A_csr @ A_csc.tocsr() - mmul_csr = csr_csc_matmul(A_csr, A_csc) - # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) - # np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) - # np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) - compare_alike(mmul_csr_native, mmul_csr) - def test_csr_csc_matmul_native_memory(cs_matrix_creator): """Check the csr_matmul function for timing and memory consumption""" A_csr, = cs_matrix_creator(nbr=1, mode='r') A_csc, = cs_matrix_creator(nbr=1, mode='c') _ = A_csr @ A_csc.tocsr() -def test_csr_csc_matmul_memory(cs_matrix_creator): - """Check the csr native @ function for timing and memory consumption""" - A_csr, = cs_matrix_creator(nbr=1, mode='r') - A_csc, = cs_matrix_creator(nbr=1, mode='c') - _ = csr_csc_matmul(A_csr, A_csc) - -def test_csr_csrT_matmul_compare(cs_matrix_creator, compare_alike): - """Compare the csr_csrT_matmul to a native implementation""" - A_csr1, = cs_matrix_creator(nbr=1, size=100000, - nbr_non_zeros=2000, mode='r') - A_csr2, = cs_matrix_creator(nbr=1, size=100000, - nbr_non_zeros=2000, mode='r') - mmul_csr_native = A_csr1 @ A_csr2.T - mmul_csr = csr_csrT_matmul(A_csr1, A_csr2) - # np.testing.assert_allclose(mmul_csr_native.data, mmul_csr.data) - # np.testing.assert_equal(mmul_csr_native.indices, mmul_csr.indices) - # np.testing.assert_equal(mmul_csr_native.indptr, mmul_csr.indptr) - compare_alike(mmul_csr_native, mmul_csr) - def test_csr_csrT_matmul_native_memory(cs_matrix_creator): """Check the csr_csrT_matmul function for timing and memory consumption""" A_csr, = cs_matrix_creator(nbr=1, mode='r') A_csc, = cs_matrix_creator(nbr=1, mode='c') _ = A_csr @ A_csc.tocsr().T -def test_csr_csrT_matmul_memory(cs_matrix_creator): - """Check the csr_csrT native @ function for timing and memory consumption""" - A_csr, = cs_matrix_creator(nbr=1, mode='r') - A_csc, = cs_matrix_creator(nbr=1, mode='c') - _ = csr_csrT_matmul(A_csr, A_csc) - # ### def test_inplace_diag_matmul_csr(cs_matrix_creator): From b10ed5c4792d3db6c79a124a89e5167fef879c0c Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Dec 2024 23:43:09 +0100 Subject: [PATCH 84/86] fixing the data type for now to double --- src/flowstab/SparseStochMat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flowstab/SparseStochMat.py b/src/flowstab/SparseStochMat.py index b33eaeb..16c2a24 100755 --- a/src/flowstab/SparseStochMat.py +++ b/src/flowstab/SparseStochMat.py @@ -195,7 +195,7 @@ def from_full_csr_matrix(cls, Tcsr:csr_matrix, nz_rowcols:NDArray|None=None, res = _css.sparse_stoch_from_full_csr( np.array(nz_rowcols, dtype=np.int32), - Tcsr.data, + Tcsr.data.astype(dtype=np.float64), Tcsr.indices, Tcsr.indptr, diag_val) From 56521ad547c892366cc66756da2f424ee14618fa Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Dec 2024 23:50:33 +0100 Subject: [PATCH 85/86] adapting lifetime of modified objects --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 355baa7..baa72c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,7 @@ def get_csr_matrix_large(): density = nbr_non_zeros / size return csr_matrix((data, (row, col)), shape=(size, size)), density -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def cs_matrix_creator(): """Creat an exemplary csr matrix that can be used for testing """ @@ -50,7 +50,7 @@ def _get_matrix(nbr:int=1,size:int=size, nbr_non_zeros:int=nbr_non_zeros, mode=' return tuple(matrices) return _get_matrix -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def SSM_matrix_creator(): """Creat an exemplary csr matrix that can be used for testing """ From f67781a4165da5f9df9f14fee58d4ea5ba2284c4 Mon Sep 17 00:00:00 2001 From: "Jonas I. Liechti" Date: Mon, 16 Dec 2024 23:51:01 +0100 Subject: [PATCH 86/86] adapting to new definitions --- tests/test_sparse_stoch_matrix.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/test_sparse_stoch_matrix.py b/tests/test_sparse_stoch_matrix.py index 5336e36..ddd321a 100644 --- a/tests/test_sparse_stoch_matrix.py +++ b/tests/test_sparse_stoch_matrix.py @@ -242,11 +242,6 @@ def test_inplace_diag_matmul_csr(cs_matrix_creator): # ### # Testing the autocovaraince matrix class # ### -def test_SparseAutocovMatrixCSR(): - """Check basic operations on sparse_autocov_csr_mat""" - from flowstab.SparseStochMat import sparse_autocov_csr_mat as SAMCSR - # ignoring this for now - pass @pytest.mark.parametrize("p1, p2, size", [(np.random.random(size=1000), @@ -328,17 +323,17 @@ def test_SAM_from_T_forward(p1, p2, size, cs_matrix_creator): SAM.from_T_forward(T=T, p1=p1, p2=p2).PT.data ) -def test_sparse_matmul_mkl_memory(csr_matrix_creator): +def test_sparse_matmul_mkl_memory(cs_matrix_creator): """ """ from sparse_dot_mkl import dot_product_mkl as mkl_matmul - A, B = csr_matrix_creator(nbr=2) + A, B = cs_matrix_creator(nbr=2) for _ in range(1000): _ = mkl_matmul(A, B) -def test_sparse_matmul_memory(csr_matrix_creator): +def test_sparse_matmul_memory(cs_matrix_creator): """ """ - A, B = csr_matrix_creator(nbr=2) + A, B = cs_matrix_creator(nbr=2) for _ in range(1000): _ = A @ B