From e401f75c48ae6c5774cec230c84841d10ee8050e Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Tue, 11 Jun 2024 16:34:31 -0400 Subject: [PATCH 1/5] Add is3D setting to summary_images functions --- caiman/summary_images.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/caiman/summary_images.py b/caiman/summary_images.py index f9a841d35..7bb50a108 100644 --- a/caiman/summary_images.py +++ b/caiman/summary_images.py @@ -591,7 +591,8 @@ def local_correlations_movie(file_name, stride: int = 1, swap_dim: bool = False, eight_neighbours: bool = True, - mode: str = 'simple'): + mode: str = 'simple', + is3D: bool = False): """ Compute an online correlation image as moving average @@ -614,13 +615,15 @@ def local_correlations_movie(file_name, Use 18 neighbors if true, and 6 if false for 4D data mode: 'simple', 'exponential', or 'cumulative' Mode of moving average + is3D: Boolean + Whether the movie has 3 spatial dimensions Returns: corr_movie: caiman.movie (3D or 4D). local correlation movie """ - Y = caiman.load(file_name) if isinstance(file_name, str) else file_name + Y = caiman.load(file_name, is3D=is3D) if isinstance(file_name, str) else file_name Y = Y[..., :tot_frames] if swap_dim else Y[:tot_frames] first_moment, second_moment, crosscorr, col_ind, row_ind, num_neigbors, M, cn = \ prepare_local_correlations(Y[..., :window] if swap_dim else Y[:window], @@ -662,7 +665,8 @@ def local_correlations_movie_offline(file_name, remove_baseline: bool = False, winSize_baseline: int = 50, quantil_min_baseline: float = 8, - gaussian_blur: bool=False): + gaussian_blur: bool=False, + is3D: bool = False): """ Efficient (parallel) computation of correlation image in shifting windows with option for prior baseline removal @@ -705,6 +709,9 @@ def local_correlations_movie_offline(file_name, gaussian_blur: bool (False) Gaussian smooth the signal + + is3D: bool (False) + Whether movie has 3 spatial dimensions Returns: mm: caiman.movie (3D or 4D). @@ -716,12 +723,12 @@ def local_correlations_movie_offline(file_name, params:list = [[file_name, range(j, j + window), eight_neighbours, swap_dim, order_mean, ismulticolor, remove_baseline, winSize_baseline, - quantil_min_baseline, gaussian_blur] + quantil_min_baseline, gaussian_blur, is3D] for j in range(0, Tot_frames - window, stride)] params.append([file_name, range(Tot_frames - window, Tot_frames), eight_neighbours, swap_dim, order_mean, ismulticolor, remove_baseline, winSize_baseline, - quantil_min_baseline, gaussian_blur]) + quantil_min_baseline, gaussian_blur, is3D]) if dview is None: parallel_result = list(map(local_correlations_movie_parallel, params)) @@ -738,8 +745,9 @@ def local_correlations_movie_offline(file_name, def local_correlations_movie_parallel(params:tuple) -> np.ndarray: - mv_name, idx, eight_neighbours, swap_dim, order_mean, ismulticolor, remove_baseline, winSize_baseline, quantil_min_baseline, gaussian_blur = params - mv = caiman.load(mv_name, subindices=idx, in_memory=True) + (mv_name, idx, eight_neighbours, swap_dim, order_mean, ismulticolor, + remove_baseline, winSize_baseline, quantil_min_baseline, gaussian_blur, is3D) = params + mv = caiman.load(mv_name, subindices=idx, in_memory=True, is3D=is3D) if gaussian_blur: mv = mv.gaussian_blur_2D() @@ -756,7 +764,8 @@ def mean_image(file_name, Tot_frames=None, fr: float = 10., window: int = 100, - dview=None): + dview=None, + is3D: bool = False): """ Efficient (parallel) computation of mean image in chunks @@ -775,21 +784,24 @@ def mean_image(file_name, dview: map object Use it for parallel computation + + is3D: bool (False) + Whether movie has 3 spatial dimensions Returns: - mm: caiman.movie (2D). + mm: caiman.movie (2D or 3D). mean image """ if Tot_frames is None: _, Tot_frames = caiman.base.movies.get_file_size(file_name) - params:list = [[file_name, range(j * window, (j + 1) * window)] + params:list = [[file_name, range(j * window, (j + 1) * window), is3D] for j in range(int(Tot_frames / window))] remain_frames = Tot_frames - int(Tot_frames / window) * window if remain_frames > 0: - params.append([file_name, range(int(Tot_frames / window) * window, Tot_frames)]) + params.append([file_name, range(int(Tot_frames / window) * window, Tot_frames), is3D]) if dview is None: parallel_result = list(map(mean_image_parallel, params)) @@ -808,6 +820,6 @@ def mean_image(file_name, return mean_image def mean_image_parallel(params:tuple) -> np.ndarray: - mv_name, idx = params - mv = caiman.load(mv_name, subindices=idx, in_memory=True) + mv_name, idx, is3D = params + mv = caiman.load(mv_name, subindices=idx, in_memory=True, is3D=is3D) return mv.mean(axis=0)[np.newaxis,:,:] From af098769baccda2801f1bdc51fe200b080865fb7 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Wed, 12 Jun 2024 21:54:25 -0400 Subject: [PATCH 2/5] Make movie.removeBL work with 3D movies --- caiman/base/movies.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/caiman/base/movies.py b/caiman/base/movies.py index 9f44dac43..1dd85551a 100644 --- a/caiman/base/movies.py +++ b/caiman/base/movies.py @@ -552,15 +552,15 @@ def removeBL(self, windowSize:int=100, quantilMin:int=8, in_place:bool=False, re def to2DPixelxTime(self, order='F'): """ - Transform 3D movie into 2D + Transform 3D or 4D movie into 2D """ - return self.transpose([2,1,0]).reshape((-1,self.shape[0]),order=order) + return self.transpose().reshape((-1,self.shape[0]),order=order) def to3DFromPixelxTime(self, shape, order='F'): """ - Transform 2D movie into 3D + Transform 2D movie into 3D or 4D """ - return to_3D(self,shape[::-1],order=order).transpose([2,1,0]) + return to_3D(self,shape[::-1],order=order).transpose() def computeDFF(self, secsWindow: int = 5, quantilMin: int = 8, method: str = 'only_baseline', in_place: bool = False, order: str = 'F') -> tuple[Any, Any]: From d13958884dacfa5768d89094f57624610ea413b6 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Fri, 14 Jun 2024 10:42:46 -0400 Subject: [PATCH 3/5] Revert "Add is3D setting to summary_images functions" This reverts commit e401f75c48ae6c5774cec230c84841d10ee8050e. --- caiman/summary_images.py | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/caiman/summary_images.py b/caiman/summary_images.py index 7bb50a108..f9a841d35 100644 --- a/caiman/summary_images.py +++ b/caiman/summary_images.py @@ -591,8 +591,7 @@ def local_correlations_movie(file_name, stride: int = 1, swap_dim: bool = False, eight_neighbours: bool = True, - mode: str = 'simple', - is3D: bool = False): + mode: str = 'simple'): """ Compute an online correlation image as moving average @@ -615,15 +614,13 @@ def local_correlations_movie(file_name, Use 18 neighbors if true, and 6 if false for 4D data mode: 'simple', 'exponential', or 'cumulative' Mode of moving average - is3D: Boolean - Whether the movie has 3 spatial dimensions Returns: corr_movie: caiman.movie (3D or 4D). local correlation movie """ - Y = caiman.load(file_name, is3D=is3D) if isinstance(file_name, str) else file_name + Y = caiman.load(file_name) if isinstance(file_name, str) else file_name Y = Y[..., :tot_frames] if swap_dim else Y[:tot_frames] first_moment, second_moment, crosscorr, col_ind, row_ind, num_neigbors, M, cn = \ prepare_local_correlations(Y[..., :window] if swap_dim else Y[:window], @@ -665,8 +662,7 @@ def local_correlations_movie_offline(file_name, remove_baseline: bool = False, winSize_baseline: int = 50, quantil_min_baseline: float = 8, - gaussian_blur: bool=False, - is3D: bool = False): + gaussian_blur: bool=False): """ Efficient (parallel) computation of correlation image in shifting windows with option for prior baseline removal @@ -709,9 +705,6 @@ def local_correlations_movie_offline(file_name, gaussian_blur: bool (False) Gaussian smooth the signal - - is3D: bool (False) - Whether movie has 3 spatial dimensions Returns: mm: caiman.movie (3D or 4D). @@ -723,12 +716,12 @@ def local_correlations_movie_offline(file_name, params:list = [[file_name, range(j, j + window), eight_neighbours, swap_dim, order_mean, ismulticolor, remove_baseline, winSize_baseline, - quantil_min_baseline, gaussian_blur, is3D] + quantil_min_baseline, gaussian_blur] for j in range(0, Tot_frames - window, stride)] params.append([file_name, range(Tot_frames - window, Tot_frames), eight_neighbours, swap_dim, order_mean, ismulticolor, remove_baseline, winSize_baseline, - quantil_min_baseline, gaussian_blur, is3D]) + quantil_min_baseline, gaussian_blur]) if dview is None: parallel_result = list(map(local_correlations_movie_parallel, params)) @@ -745,9 +738,8 @@ def local_correlations_movie_offline(file_name, def local_correlations_movie_parallel(params:tuple) -> np.ndarray: - (mv_name, idx, eight_neighbours, swap_dim, order_mean, ismulticolor, - remove_baseline, winSize_baseline, quantil_min_baseline, gaussian_blur, is3D) = params - mv = caiman.load(mv_name, subindices=idx, in_memory=True, is3D=is3D) + mv_name, idx, eight_neighbours, swap_dim, order_mean, ismulticolor, remove_baseline, winSize_baseline, quantil_min_baseline, gaussian_blur = params + mv = caiman.load(mv_name, subindices=idx, in_memory=True) if gaussian_blur: mv = mv.gaussian_blur_2D() @@ -764,8 +756,7 @@ def mean_image(file_name, Tot_frames=None, fr: float = 10., window: int = 100, - dview=None, - is3D: bool = False): + dview=None): """ Efficient (parallel) computation of mean image in chunks @@ -784,24 +775,21 @@ def mean_image(file_name, dview: map object Use it for parallel computation - - is3D: bool (False) - Whether movie has 3 spatial dimensions Returns: - mm: caiman.movie (2D or 3D). + mm: caiman.movie (2D). mean image """ if Tot_frames is None: _, Tot_frames = caiman.base.movies.get_file_size(file_name) - params:list = [[file_name, range(j * window, (j + 1) * window), is3D] + params:list = [[file_name, range(j * window, (j + 1) * window)] for j in range(int(Tot_frames / window))] remain_frames = Tot_frames - int(Tot_frames / window) * window if remain_frames > 0: - params.append([file_name, range(int(Tot_frames / window) * window, Tot_frames), is3D]) + params.append([file_name, range(int(Tot_frames / window) * window, Tot_frames)]) if dview is None: parallel_result = list(map(mean_image_parallel, params)) @@ -820,6 +808,6 @@ def mean_image(file_name, return mean_image def mean_image_parallel(params:tuple) -> np.ndarray: - mv_name, idx, is3D = params - mv = caiman.load(mv_name, subindices=idx, in_memory=True, is3D=is3D) + mv_name, idx = params + mv = caiman.load(mv_name, subindices=idx, in_memory=True) return mv.mean(axis=0)[np.newaxis,:,:] From 0857d7aae5f285e5da35ee81da50bc4abf25e0d6 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Fri, 14 Jun 2024 11:24:21 -0400 Subject: [PATCH 4/5] Add tests for to2D and to3D --- .gitignore | 1 + caiman/base/movies.py | 12 ++++++------ caiman/tests/test_movies.py | 25 ++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 147a0db9c..03bbb9705 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,7 @@ __pycache__/ # C extensions *.so *.c +*.cpp #movie tif *.tif diff --git a/caiman/base/movies.py b/caiman/base/movies.py index 1dd85551a..2cfb3d245 100644 --- a/caiman/base/movies.py +++ b/caiman/base/movies.py @@ -550,17 +550,17 @@ def removeBL(self, windowSize:int=100, quantilMin:int=8, in_place:bool=False, re self -= caiman.movie(cv2.resize(res,pixs.shape[::-1]),fr=self.fr).to3DFromPixelxTime(self.shape) return self - def to2DPixelxTime(self, order='F'): + def to2DPixelxTime(self, order='F') -> 'movie': """ - Transform 3D or 4D movie into 2D + Transform 3D or 4D movie into 2D (pixels x time) """ - return self.transpose().reshape((-1,self.shape[0]),order=order) + return self.T.reshape((-1,self.shape[0]),order=order) - def to3DFromPixelxTime(self, shape, order='F'): + def to3DFromPixelxTime(self, shape, order='F') -> 'movie': """ - Transform 2D movie into 3D or 4D + Transform 2D movie (pixels x time) into 3D or 4D """ - return to_3D(self,shape[::-1],order=order).transpose() + return to_3D(self,shape[::-1],order=order).T def computeDFF(self, secsWindow: int = 5, quantilMin: int = 8, method: str = 'only_baseline', in_place: bool = False, order: str = 'F') -> tuple[Any, Any]: diff --git a/caiman/tests/test_movies.py b/caiman/tests/test_movies.py index 2287307f9..35e8c6f9f 100644 --- a/caiman/tests/test_movies.py +++ b/caiman/tests/test_movies.py @@ -2,18 +2,37 @@ import numpy as np import numpy.testing as npt import os -from caiman.base.movies import load, load_iter +from caiman.base import movies from caiman.paths import caiman_datadir def test_load_iter(): fname = os.path.join(caiman_datadir(), 'example_movies', 'demoMovie.tif') for subindices in (None, slice(100, None, 2)): - m = load_iter(fname, subindices=subindices) + m = movies.load_iter(fname, subindices=subindices) S = 0 while True: try: S += np.sum(next(m)) except StopIteration: break - npt.assert_allclose(S, load(fname, subindices=subindices).sum(), rtol=1e-6) + npt.assert_allclose(S, movies.load(fname, subindices=subindices).sum(), rtol=1e-6) + + +def test_to2D_to3D(): + def reference_to2D(mov, order: str): + return mov.T.reshape((-1, mov.shape[0]), order=order) + + # set up small test movies + rng = np.random.default_rng() + T, y, x, z = 10, 6, 4, 2 + movie_3D = movies.movie(rng.random((T, y, x))) + movie_4D = movies.movie(rng.random((T, y, x, z))) + + for movie in (movie_3D, movie_4D): + for order in ('F', 'C'): + shape = movie.shape + movie_flat = movie.to2DPixelxTime(order=order) + npt.assert_array_equal(movie_flat, reference_to2D(movie, order)) + movie_from_flat = movie_flat.to3DFromPixelxTime(shape, order) + npt.assert_array_equal(movie_from_flat, movie) From 7e372b397268a04fc788a7e91dcfd5d0ae267227 Mon Sep 17 00:00:00 2001 From: Ethan Blackwood Date: Sun, 16 Jun 2024 14:04:54 -0400 Subject: [PATCH 5/5] Make cpp exclusion more specific --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 03bbb9705..b6f9b597e 100644 --- a/.gitignore +++ b/.gitignore @@ -52,8 +52,7 @@ __pycache__/ # C extensions *.so -*.c -*.cpp +caiman/source_extraction/cnmf/oasis.cpp #movie tif *.tif