From 9a18065be4202292fb73ca8566d39dc232751285 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Wed, 12 Jun 2024 01:34:31 -0400 Subject: [PATCH 1/6] Fix failure to compute correlation image for 3D data (pending caiman update) --- mesmerize_core/algorithms/mcorr.py | 52 ++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/mesmerize_core/algorithms/mcorr.py b/mesmerize_core/algorithms/mcorr.py index 755f697..e75f4e5 100644 --- a/mesmerize_core/algorithms/mcorr.py +++ b/mesmerize_core/algorithms/mcorr.py @@ -10,6 +10,9 @@ import numpy as np from shutil import move as move_file import time +import pkg_resources +from packaging.version import Version + # prevent circular import if __name__ in ["__main__", "__mp_main__"]: # when running in subprocess @@ -88,25 +91,30 @@ def run_algo(batch_path, uuid, data_path: str = None): ) np.save(str(proj_paths[proj_type]), p_img) - print("Computing correlation image") - Cns = local_correlations_movie_offline( - [str(mcorr_memmap_path)], - remove_baseline=True, - window=1000, - stride=1000, - winSize_baseline=100, - quantil_min_baseline=10, - dview=dview, - ) - Cn = Cns.max(axis=0) - Cn[np.isnan(Cn)] = 0 - cn_path = output_dir.joinpath(f"{uuid}_cn.npy") - np.save(str(cn_path), Cn, allow_pickle=False) - - # output dict for pandas series for dataframe row - d = dict() - - print("finished computing correlation image") + is3D = len(dims) == 3 + if is3D and Version(pkg_resources.get_distribution('caiman').version) < Version('1.11.2'): + # is3D parameter only (to be) added in version 1.11.2 + print("Skipping correlation image of 3D movie (not supported)") + cn_path = None + else: + print("Computing correlation image") + kwargs = {'is3D': True} if is3D else {} + Cns = local_correlations_movie_offline( + [str(mcorr_memmap_path)], + remove_baseline=True, + window=1000, + stride=1000, + winSize_baseline=100, + quantil_min_baseline=10, + dview=dview, + **kwargs + ) + Cn = Cns.max(axis=0) + Cn[np.isnan(Cn)] = 0 + cn_path = output_dir.joinpath(f"{uuid}_cn.npy") + np.save(str(cn_path), Cn, allow_pickle=False) + + print("finished computing correlation image") # Compute shifts if opts.motion["pw_rigid"] == True: @@ -120,8 +128,12 @@ def run_algo(batch_path, uuid, data_path: str = None): shift_path = output_dir.joinpath(f"{uuid}_shifts.npy") np.save(str(shift_path), shifts) + # output dict for pandas series for dataframe row + d = dict() + # save paths as relative path strings with forward slashes - cn_path = str(PurePosixPath(cn_path.relative_to(output_dir.parent))) + if cn_path is not None: + cn_path = str(PurePosixPath(cn_path.relative_to(output_dir.parent))) mcorr_memmap_path = str(PurePosixPath(mcorr_memmap_path.relative_to(output_dir.parent))) shift_path = str(PurePosixPath(shift_path.relative_to(output_dir.parent))) for proj_type in proj_paths.keys(): From 7bcb8f6a7b4a27bb6cbafc0a6effb375be43b9ab Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Fri, 14 Jun 2024 12:18:17 -0400 Subject: [PATCH 2/6] Fix by passing file as string and re-run without baseline if that fails --- mesmerize_core/algorithms/mcorr.py | 34 ++++++++++++++++-------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/mesmerize_core/algorithms/mcorr.py b/mesmerize_core/algorithms/mcorr.py index e75f4e5..72cab04 100644 --- a/mesmerize_core/algorithms/mcorr.py +++ b/mesmerize_core/algorithms/mcorr.py @@ -10,9 +10,6 @@ import numpy as np from shutil import move as move_file import time -import pkg_resources -from packaging.version import Version - # prevent circular import if __name__ in ["__main__", "__mp_main__"]: # when running in subprocess @@ -91,23 +88,15 @@ def run_algo(batch_path, uuid, data_path: str = None): ) np.save(str(proj_paths[proj_type]), p_img) - is3D = len(dims) == 3 - if is3D and Version(pkg_resources.get_distribution('caiman').version) < Version('1.11.2'): - # is3D parameter only (to be) added in version 1.11.2 - print("Skipping correlation image of 3D movie (not supported)") - cn_path = None - else: - print("Computing correlation image") - kwargs = {'is3D': True} if is3D else {} + def run_correlation(remove_baseline=True) -> Path: Cns = local_correlations_movie_offline( - [str(mcorr_memmap_path)], - remove_baseline=True, + str(mcorr_memmap_path), + remove_baseline=remove_baseline, window=1000, stride=1000, winSize_baseline=100, quantil_min_baseline=10, dview=dview, - **kwargs ) Cn = Cns.max(axis=0) Cn[np.isnan(Cn)] = 0 @@ -115,6 +104,20 @@ def run_algo(batch_path, uuid, data_path: str = None): np.save(str(cn_path), Cn, allow_pickle=False) print("finished computing correlation image") + return cn_path + + try: + print("Computing correlation image") + cn_path = run_correlation() + + except ValueError as err: + # Test for error that occurs in movie.removeBL before bug was fixed + is3D = len(dims) == 3 + if is3D and len(err.args) == 1 and err.args[0] == "axes don't match array": + print("Computing correlation on 3D image failed - trying without baseline (use caiman >= 1.11.2 to fix)") + cn_path = run_correlation(remove_baseline=False) + else: + raise # Compute shifts if opts.motion["pw_rigid"] == True: @@ -132,8 +135,7 @@ def run_algo(batch_path, uuid, data_path: str = None): d = dict() # save paths as relative path strings with forward slashes - if cn_path is not None: - cn_path = str(PurePosixPath(cn_path.relative_to(output_dir.parent))) + cn_path = str(PurePosixPath(cn_path.relative_to(output_dir.parent))) mcorr_memmap_path = str(PurePosixPath(mcorr_memmap_path.relative_to(output_dir.parent))) shift_path = str(PurePosixPath(shift_path.relative_to(output_dir.parent))) for proj_type in proj_paths.keys(): From e301d636d03f1105587adff738e3fdbfa6d685d4 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Sat, 22 Jun 2024 15:52:36 -0400 Subject: [PATCH 3/6] Make local_correlations in cnmf compatible with 3D data --- mesmerize_core/algorithms/cnmf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesmerize_core/algorithms/cnmf.py b/mesmerize_core/algorithms/cnmf.py index 4c71174..ca8f599 100644 --- a/mesmerize_core/algorithms/cnmf.py +++ b/mesmerize_core/algorithms/cnmf.py @@ -99,7 +99,7 @@ def run_algo(batch_path, uuid, data_path: str = None): cnm.save(str(output_path)) - Cn = cm.local_correlations(images.transpose(1, 2, 0)) + Cn = cm.local_correlations(images, swap_dim=False) Cn[np.isnan(Cn)] = 0 corr_img_path = output_dir.joinpath(f"{uuid}_cn.npy").resolve() From 0a318b36d0862e75fa8e8543c50ae189de8d0634 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Wed, 24 Jul 2024 15:00:07 -0400 Subject: [PATCH 4/6] Add z shifts for pw_rigid case --- mesmerize_core/algorithms/mcorr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mesmerize_core/algorithms/mcorr.py b/mesmerize_core/algorithms/mcorr.py index 72cab04..47f9e45 100644 --- a/mesmerize_core/algorithms/mcorr.py +++ b/mesmerize_core/algorithms/mcorr.py @@ -124,6 +124,8 @@ def run_correlation(remove_baseline=True) -> Path: x_shifts = mc.x_shifts_els y_shifts = mc.y_shifts_els shifts = [x_shifts, y_shifts] + if hasattr(mc, 'z_shifts_els'): + shifts += mc.z_shifts_els shift_path = output_dir.joinpath(f"{uuid}_shifts.npy") np.save(str(shift_path), shifts) else: From 9454803510b1786f1c65506537a6751509acb719 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Wed, 24 Jul 2024 15:57:53 -0400 Subject: [PATCH 5/6] Pin caiman version instead of catching error from old version --- environment.yml | 2 +- environment_rtd.yml | 2 +- mesmerize_core/algorithms/mcorr.py | 45 +++++++++++------------------- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/environment.yml b/environment.yml index 5caddb9..aa793aa 100644 --- a/environment.yml +++ b/environment.yml @@ -1,7 +1,7 @@ channels: - conda-forge dependencies: - - caiman >= 1.9.10 + - caiman >= 1.11.2 - pandas >= 1.5.0 - requests - click diff --git a/environment_rtd.yml b/environment_rtd.yml index b51dbea..ce94dbc 100644 --- a/environment_rtd.yml +++ b/environment_rtd.yml @@ -1,7 +1,7 @@ channels: - conda-forge dependencies: - - caiman >= 1.9.10 + - caiman >= 1.11.2 - pandas >= 1.5.0 - requests - click diff --git a/mesmerize_core/algorithms/mcorr.py b/mesmerize_core/algorithms/mcorr.py index 47f9e45..fbfd1fd 100644 --- a/mesmerize_core/algorithms/mcorr.py +++ b/mesmerize_core/algorithms/mcorr.py @@ -88,36 +88,23 @@ def run_algo(batch_path, uuid, data_path: str = None): ) np.save(str(proj_paths[proj_type]), p_img) - def run_correlation(remove_baseline=True) -> Path: - Cns = local_correlations_movie_offline( - str(mcorr_memmap_path), - remove_baseline=remove_baseline, - window=1000, - stride=1000, - winSize_baseline=100, - quantil_min_baseline=10, - dview=dview, - ) - Cn = Cns.max(axis=0) - Cn[np.isnan(Cn)] = 0 - cn_path = output_dir.joinpath(f"{uuid}_cn.npy") - np.save(str(cn_path), Cn, allow_pickle=False) - - print("finished computing correlation image") - return cn_path + print("Computing correlation image") + Cns = local_correlations_movie_offline( + str(mcorr_memmap_path), + remove_baseline=True, + window=1000, + stride=1000, + winSize_baseline=100, + quantil_min_baseline=10, + dview=dview, + ) + Cn = Cns.max(axis=0) + Cn[np.isnan(Cn)] = 0 + cn_path = output_dir.joinpath(f"{uuid}_cn.npy") + np.save(str(cn_path), Cn, allow_pickle=False) + + print("finished computing correlation image") - try: - print("Computing correlation image") - cn_path = run_correlation() - - except ValueError as err: - # Test for error that occurs in movie.removeBL before bug was fixed - is3D = len(dims) == 3 - if is3D and len(err.args) == 1 and err.args[0] == "axes don't match array": - print("Computing correlation on 3D image failed - trying without baseline (use caiman >= 1.11.2 to fix)") - cn_path = run_correlation(remove_baseline=False) - else: - raise # Compute shifts if opts.motion["pw_rigid"] == True: From 0245a1fc0ab41c7a95c63bf9d389fcc506e19ed5 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Sun, 1 Sep 2024 00:29:38 -0400 Subject: [PATCH 6/6] Fix saving non-rigid z-shifts --- mesmerize_core/algorithms/mcorr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mesmerize_core/algorithms/mcorr.py b/mesmerize_core/algorithms/mcorr.py index fbfd1fd..0ba3793 100644 --- a/mesmerize_core/algorithms/mcorr.py +++ b/mesmerize_core/algorithms/mcorr.py @@ -105,14 +105,13 @@ def run_algo(batch_path, uuid, data_path: str = None): print("finished computing correlation image") - # Compute shifts if opts.motion["pw_rigid"] == True: x_shifts = mc.x_shifts_els y_shifts = mc.y_shifts_els shifts = [x_shifts, y_shifts] if hasattr(mc, 'z_shifts_els'): - shifts += mc.z_shifts_els + shifts.append(mc.z_shifts_els) shift_path = output_dir.joinpath(f"{uuid}_shifts.npy") np.save(str(shift_path), shifts) else: