diff --git a/caiman/base/movies.py b/caiman/base/movies.py index 93fbecfd8..b4494d4ea 100644 --- a/caiman/base/movies.py +++ b/caiman/base/movies.py @@ -343,7 +343,7 @@ def extract_shifts(self, max_shift_w: int = 5, max_shift_h: int = 5, template=No template = template[ms_h:h_i - ms_h, ms_w:w_i - ms_w].astype(np.float32) - #% run algorithm, press q to stop it + # run algorithm, press q to stop it shifts = [] # store the amount of shift in each frame xcorrs = [] @@ -604,7 +604,7 @@ def computeDFF(self, secsWindow: int = 5, quantilMin: int = 8, method: str = 'on **self.__dict__) numFramesNew, linePerFrame, pixPerLine = np.shape(mov_out) - #% compute baseline quickly + # compute baseline quickly logging.debug("binning data ...") sys.stdout.flush() @@ -627,7 +627,7 @@ def computeDFF(self, secsWindow: int = 5, quantilMin: int = 8, method: str = 'on cval=0.0, prefilter=False) - #% compute DF/F + # compute DF/F if not in_place: if method == 'delta_f_over_sqrt_f': mov_out = (mov_out - movBL) / np.sqrt(movBL) diff --git a/caiman/base/rois.py b/caiman/base/rois.py index 5724d1be9..b5fff51e1 100644 --- a/caiman/base/rois.py +++ b/caiman/base/rois.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import cv2 import json @@ -242,7 +241,7 @@ def nf_match_neurons_in_binary_masks(masks_gt, cm_cnmf = [scipy.ndimage.center_of_mass(mm) for mm in masks_comp] if D is None: - #% find distances and matches + # find distances and matches # find the distance between each masks D = distance_masks([A_ben, A_cnmf], [cm_ben, cm_cnmf], min_dist, enclosed_thr=enclosed_thr) @@ -252,7 +251,7 @@ def nf_match_neurons_in_binary_masks(masks_gt, matches = matches[0] costs = costs[0] - #%% compute precision and recall + # compute precision and recall TP = np.sum(np.array(costs) < thresh_cost) * 1. FN = np.shape(masks_gt)[0] - TP FP = np.shape(masks_comp)[0] - TP @@ -264,7 +263,7 @@ def nf_match_neurons_in_binary_masks(masks_gt, performance['accuracy'] = (TP + TN) / (TP + FP + FN + TN) performance['f1_score'] = 2 * TP / (2 * TP + FP + FN) logging.debug(performance) - #%% + idx_tp = np.where(np.array(costs) < thresh_cost)[0] idx_tp_ben = matches[0][idx_tp] # ground truth idx_tp_cnmf = matches[1][idx_tp] # algorithm - comp @@ -471,7 +470,7 @@ def register_ROIs(A1, matches = matches[0] costs = costs[0] - #%% store indices + # store indices idx_tp = np.where(np.array(costs) < thresh_cost)[0] if len(idx_tp) > 0: @@ -488,7 +487,7 @@ def register_ROIs(A1, non_matched1 = list(range(D[0].shape[0])) non_matched2 = list(range(D[0].shape[1])) - #%% compute precision and recall + # compute precision and recall FN = D[0].shape[0] - TP FP = D[0].shape[1] - TP @@ -1001,7 +1000,7 @@ def getfloat(): fill_color = get32() subtype = get16() if subtype != 0: - raise ValueError('roireader: ROI subtype %s not supported (!= 0)' % subtype) + raise ValueError(f'roireader: ROI subtype {subtype} not supported (!= 0)') options = get16() arrow_style = get8() arrow_head_size = get8() diff --git a/caiman/base/traces.py b/caiman/base/traces.py index 1a8484e77..3ec4f6aa4 100644 --- a/caiman/base/traces.py +++ b/caiman/base/traces.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import cv2 import logging diff --git a/caiman/components_evaluation.py b/caiman/components_evaluation.py index 97922b284..6b6e353b7 100644 --- a/caiman/components_evaluation.py +++ b/caiman/components_evaluation.py @@ -437,7 +437,7 @@ def evaluate_components(Y: np.ndarray, padafter = int(np.ceil(elm_missing / 2.)) tr_tmp = np.pad(traces.T, ((padbefore, padafter), (0, 0)), mode='reflect') numFramesNew, num_traces = np.shape(tr_tmp) - #% compute baseline quickly + # compute baseline quickly logging.debug("binning data ...") tr_BL = np.reshape(tr_tmp, (downsampfact, numFramesNew // downsampfact, num_traces), order='F') tr_BL = np.percentile(tr_BL, 8, axis=0) diff --git a/caiman/mmapping.py b/caiman/mmapping.py index 1f11119a5..65dd10cf4 100644 --- a/caiman/mmapping.py +++ b/caiman/mmapping.py @@ -1,16 +1,15 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import ipyparallel as parallel from itertools import chain import logging import numpy as np import os +import pathlib import pickle import sys import tifffile from typing import Any, Optional, Union -import pathlib import caiman import caiman.paths diff --git a/caiman/motion_correction.py b/caiman/motion_correction.py index 2e14c8f4a..22721981f 100644 --- a/caiman/motion_correction.py +++ b/caiman/motion_correction.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """ diff --git a/caiman/source_extraction/cnmf/cnmf_optional_outputs.py b/caiman/source_extraction/cnmf/cnmf_optional_outputs.py index 435f45cca..83c5bd89e 100644 --- a/caiman/source_extraction/cnmf/cnmf_optional_outputs.py +++ b/caiman/source_extraction/cnmf/cnmf_optional_outputs.py @@ -1,16 +1,9 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """ Constrained Nonnegative Matrix Factorization A similar CNMF class that will output data that are not usually present in the regular CNMF process - -Created on Fri Aug 26 15:44:32 2016 - -@author: agiovann - - """ import numpy as np diff --git a/caiman/source_extraction/cnmf/deconvolution.py b/caiman/source_extraction/cnmf/deconvolution.py index 6d185c63a..5cf339678 100644 --- a/caiman/source_extraction/cnmf/deconvolution.py +++ b/caiman/source_extraction/cnmf/deconvolution.py @@ -1,10 +1,7 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- -"""Extract neural activity from a fluorescence trace using a constrained deconvolution approach - -Created on Tue Sep 1 16:11:25 2015 -@author: Eftychios A. Pnevmatikakis, based on an implementation by T. Machado, Andrea Giovannucci & Ben Deverett +""" +Extract neural activity from a fluorescence trace using a constrained deconvolution approach """ import numpy as np @@ -15,8 +12,6 @@ from math import log, sqrt, exp import sys -#%% - def constrained_foopsi(fluor, bl=None, c1=None, g=None, sn=None, p=None, method_deconvolution='oasis', bas_nonneg=True, noise_range=[.25, .5], noise_method='logmexp', lags=5, fudge_factor=1., @@ -104,9 +99,6 @@ def constrained_foopsi(fluor, bl=None, c1=None, g=None, sn=None, p=None, metho References: * Pnevmatikakis et al. 2016. Neuron, in press, http://dx.doi.org/10.1016/j.neuron.2015.11.037 * Machado et al. 2015. Cell 162(2):338-350 - - \image: docs/img/deconvolution.png - \image: docs/img/evaluationcomponent.png """ if p is None: diff --git a/caiman/source_extraction/cnmf/online_cnmf.py b/caiman/source_extraction/cnmf/online_cnmf.py index ebdb1dd43..fefb1f100 100644 --- a/caiman/source_extraction/cnmf/online_cnmf.py +++ b/caiman/source_extraction/cnmf/online_cnmf.py @@ -1375,8 +1375,6 @@ def create_frame(self, frame_cor, show_residuals=True, resize_fact=3, transpose= self.dims = self.dims[::-1] return vid_frame - -#%% def bare_initialization(Y, init_batch=1000, k=1, method_init='greedy_roi', gnb=1, gSig=[5, 5], motion_flag=False, p=1, return_object=True, **kwargs): @@ -1456,8 +1454,6 @@ def bare_initialization(Y, init_batch=1000, k=1, method_init='greedy_roi', gnb=1 except: return Ain, np.array(b_in), Cin, f_in, YrA - -#%% def seeded_initialization(Y, Ain, dims=None, init_batch=1000, order_init=None, gnb=1, p=1, return_object=True, **kwargs): @@ -1513,7 +1509,7 @@ def seeded_initialization(Y, Ain, dims=None, init_batch=1000, order_init=None, g model_comp = NMF(n_components=1, init='nndsvdar', max_iter=50) for count, idx_in in enumerate(order_init): - if count%10 == 0: + if count % 10 == 0: print(count) idx_domain = np.where(Ain[:,idx_in])[0] Ain[idx_domain,idx_in] = model_comp.fit_transform(\ @@ -1771,7 +1767,7 @@ def refit(o, c): return C, noisyC, OASISinstances -#%% Estimate shapes on small initial batch +# Estimate shapes on small initial batch def init_shapes_and_sufficient_stats(Y, A, C, b, f, W=None, b0=None, ssub_B=1, bSiz=3, downscale_matrix=None, upscale_matrix=None): # smooth the components @@ -1943,8 +1939,6 @@ def get_last_frames(self, num_frames): else: return np.concatenate([self[(self.cur - num_frames):], self[:self.cur]], axis=0) - -#%% def csc_append(a, b): """ Takes in 2 csc_matrices and appends the second one to the right of the first one. Much faster than scipy.sparse.hstack but assumes the type to be csc and overwrites @@ -2000,7 +1994,6 @@ def rank1nmf(Ypx, ain, iters=10): cin = np.maximum(cin_res, 0) return ain, cin, cin_res -#%% @profile def get_candidate_components(sv, dims, Yres_buf, min_num_trial=3, gSig=(5, 5), gHalf=(5, 5), sniper_mode=True, rval_thr=0.85, @@ -2127,8 +2120,6 @@ def get_candidate_components(sv, dims, Yres_buf, min_num_trial=3, gSig=(5, 5), return Ain, Cin, Cin_res, idx, ijsig_all, cnn_pos, local_maxima - -#%% @profile def update_num_components(t, sv, Ab, Cf, Yres_buf, Y_buf, rho_buf, dims, gSig, gSiz, ind_A, CY, CC, groups, oases, gnb=1, @@ -2371,7 +2362,7 @@ def update_num_components(t, sv, Ab, Cf, Yres_buf, Y_buf, rho_buf, return Ab, Cf, Yres_buf, rho_buf, CC, CY, ind_A, sv, groups, ind_new, ind_new_all, sv, cnn_pos -#%% remove components online +# remove components online def remove_components_online(ind_rem, gnb, Ab, use_dense, Ab_dense, AtA, CY, CC, M, N, noisyC, OASISinstances, C_on, exp_comps): @@ -2438,10 +2429,7 @@ def initialize_movie_online(Y, K, gSig, rf, stride, base_name, images = np.reshape(Yr.T, [T] + list(dims), order='F') Y = np.reshape(Yr, dims + (T,), order='F') Cn2 = caiman.local_correlations(Y) -# pl.imshow(Cn2) - #% - #% RUN ALGORITHM ON PATCHES -# pl.close('all') + # RUN ALGORITHM ON PATCHES cnm_init = caiman.source_extraction.cnmf.CNMF(n_processes, method_init='greedy_roi', k=K, gSig=gSig, merge_thresh=merge_thresh, p=0, dview=dview, Ain=None, rf=rf, stride=stride, method_deconvolution='oasis', skip_refinement=False, normalize_init=False, options_local_NMF=None, @@ -2456,9 +2444,7 @@ def initialize_movie_online(Y, K, gSig, rf, stride, base_name, b_tot = cnm_init.b f_tot = cnm_init.f - print(('Number of components:' + str(A_tot.shape[-1]))) - - #% + print(f"Number of components: {A_tot.shape[-1]}") traces = C_tot + YrA_tot fitness_raw, fitness_delta, erfc_raw, erfc_delta, r_values, significant_samples = caiman.components_evaluation.evaluate_components( @@ -2478,7 +2464,7 @@ def initialize_movie_online(Y, K, gSig, rf, stride, base_name, A_tot = A_tot.tocsc()[:, idx_components] C_tot = C_tot[idx_components] - #% + cnm_refine = caiman.source_extraction.cnmf.CNMF(n_processes, method_init='greedy_roi', k=A_tot.shape, gSig=gSig, merge_thresh=merge_thresh, rf=None, stride=None, p=p, dview=dview, Ain=A_tot, Cin=C_tot, f_in=f_tot, method_deconvolution='oasis', skip_refinement=True, normalize_init=False, options_local_NMF=None, @@ -2487,9 +2473,9 @@ def initialize_movie_online(Y, K, gSig, rf, stride, base_name, batch_update_suff_stat=True, max_comp_update_shape=5) cnm_refine = cnm_refine.fit(images) - #% + A, C, b, f, YrA = cnm_refine.A, cnm_refine.C, cnm_refine.b, cnm_refine.f, cnm_refine.YrA - #% + final_frate = 10 Npeaks = 10 traces = C + YrA @@ -2510,7 +2496,7 @@ def initialize_movie_online(Y, K, gSig, rf, stride, base_name, print(' ***** ') print((len(traces))) print((len(idx_components))) - #% + cnm_refine.sn = sn # FIXME: There is no sn in scope here cnm_refine.idx_components = idx_components cnm_refine.idx_components_bad = idx_components_bad @@ -2519,11 +2505,6 @@ def initialize_movie_online(Y, K, gSig, rf, stride, base_name, cnm_refine.fitness_delta = fitness_delta cnm_refine.Cn2 = Cn2 - #% - -# cnm_init.dview = None -# save_object(cnm_init,fls[0][:-4]+ '_DS_' + str(ds)+ '_init.pkl') - return cnm_refine, Cn2, fname_new def load_OnlineCNMF(filename, dview = None): diff --git a/caiman/source_extraction/cnmf/utilities.py b/caiman/source_extraction/cnmf/utilities.py index 1fd88ad0f..3870bc93b 100644 --- a/caiman/source_extraction/cnmf/utilities.py +++ b/caiman/source_extraction/cnmf/utilities.py @@ -580,7 +580,7 @@ def fast_prct_filt(input_data, level=8, frames_window=1000): padafter = int(np.ceil(elm_missing / 2.)) tr_tmp = np.pad(data.T, ((padbefore, padafter), (0, 0)), mode='reflect') numFramesNew, num_traces = np.shape(tr_tmp) - #% compute baseline quickly + # compute baseline quickly tr_BL = np.reshape(tr_tmp, (downsampfact, int(numFramesNew / downsampfact), num_traces), order='F') @@ -596,7 +596,7 @@ def fast_prct_filt(input_data, level=8, frames_window=1000): data -= tr_BL[padbefore:-padafter].T return data.squeeze() -#%% + def detrend_df_f_auto(A, b, C, f, dims=None, YrA=None, use_annulus = True, dist1 = 7, dist2 = 5, frames_window=1000, use_fast = False): @@ -699,9 +699,6 @@ def detrend_df_f_auto(A, b, C, f, dims=None, YrA=None, use_annulus = True, return F_df -#%% - - def manually_refine_components(Y, xxx_todo_changeme, A, C, Cn, thr=0.9, display_numbers=True, max_number=None, cmap=None, **kwargs): """Plots contour of spatial components @@ -1014,8 +1011,6 @@ def update_order_greedy(A, flag_AA=True): parllcomp.append([i]) len_parrllcomp = [len(ls) for ls in parllcomp] return parllcomp, len_parrllcomp -#%% - def compute_residuals(Yr_mmap_file, A_, b_, C_, f_, dview=None, block_size=1000, num_blocks_per_run=5): '''compute residuals from memory mapped file and output of CNMF diff --git a/caiman/utils/labelling.py b/caiman/utils/labelling.py index abcd2ee3e..82124b4af 100644 --- a/caiman/utils/labelling.py +++ b/caiman/utils/labelling.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- import logging from scipy.ndimage import filters as ft + import caiman def pre_preprocess_movie_labeling(dview, file_names, median_filter_size=(2, 1, 1), diff --git a/caiman/utils/utils.py b/caiman/utils/utils.py index 3c5bc88cf..6b48467a3 100644 --- a/caiman/utils/utils.py +++ b/caiman/utils/utils.py @@ -43,9 +43,6 @@ import caiman.source_extraction.cnmf.spatial import caiman.utils -#%% - - def download_demo(name:str='Sue_2x_3000_40_-46.tif', save_folder:str='') -> str: """download a file from the file list with the url of its location @@ -243,7 +240,7 @@ def get_image_description_SI(fname:str) -> list: return image_descriptions -#%% Generate data +# Generate data def gen_data(dims:tuple[int,int]=(48, 48), N:int=10, sig:tuple[int,int]=(3, 3), tau:float=1., noise:float=.3, T:int=2000, framerate:int=30, firerate:float=.5, seed:int=3, cmap:bool=False, truncate:float=np.exp(-2), difference_of_Gaussians:bool=True, fluctuating_bkgrd:list=[50, 300]) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray, tuple[int, int]]: @@ -338,8 +335,6 @@ def gen_data(dims:tuple[int,int]=(48, 48), N:int=10, sig:tuple[int,int]=(3, 3), plt.show() return Yr, trueC, trueS, trueA, trueb, truef, centers, dims # XXX dims is always the same as passed into the function? - -#%% def save_object(obj, filename:str) -> None: with open(filename, 'wb') as output: pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL) @@ -349,7 +344,7 @@ def load_object(filename:str) -> Any: with open(filename, 'rb') as input_obj: obj = pickle.load(input_obj) return obj -#%% + def apply_magic_wand(A, gSig, dims, A_thr=None, coms=None, dview=None, min_frac=0.7, max_frac=1.0, roughness=2, zoom_factor=1, center_range=2) -> np.ndarray: @@ -427,7 +422,6 @@ def cell_magic_wand_wrapper(params): zoom_factor, center_range) return msk -#%% From https://codereview.stackexchange.com/questions/120802/recursively-save-python-dictionaries-to-hdf5-files-using-h5py def save_dict_to_hdf5(dic:dict, filename:str, subdir:str='/') -> None: @@ -438,6 +432,7 @@ def save_dict_to_hdf5(dic:dict, filename:str, subdir:str='/') -> None: filename: str file name to save the dictionary to (in hdf5 format for now) ''' + # From https://codereview.stackexchange.com/questions/120802/recursively-save-python-dictionaries-to-hdf5-files-using-h5py with h5py.File(filename, 'w') as h5file: recursively_save_dict_contents_to_group(h5file, subdir, dic)