From 04a04aea970ff56ee71c5cdc88091296d81e59fe Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Thu, 25 Jul 2024 11:44:29 -0400 Subject: [PATCH] Logging cleanup diffset: Get rid of almost all calls to global logger --- caiman/tests/comparison/comparison.py | 42 +++++++-------------- caiman/tests/comparison_general.py | 31 ++++++--------- caiman/tests/test_deconvolution.py | 5 ++- demos/general/demo_OnACID.py | 2 +- demos/general/demo_OnACID_mesoscope.py | 2 +- demos/notebooks/demo_OnACID_mesoscope.ipynb | 2 +- 6 files changed, 32 insertions(+), 52 deletions(-) diff --git a/caiman/tests/comparison/comparison.py b/caiman/tests/comparison/comparison.py index d866bd3aa..18d824589 100644 --- a/caiman/tests/comparison/comparison.py +++ b/caiman/tests/comparison/comparison.py @@ -23,7 +23,8 @@ # You can log to a file using the filename parameter, or make the output more or less # verbose by setting level to logging.DEBUG, logging.INFO, logging.WARNING, or logging.ERROR -logging.getLogger("caiman").setLevel(logging.DEBUG) +logger = logging.getLogger("caiman") +logger.setLevel(logging.DEBUG) import caiman as cm from caiman.paths import caiman_datadir @@ -190,7 +191,7 @@ def save_with_compare(self, istruth=False, params=None, dview=None, Cn=None): """ # getting the DATA FOR COMPARISONS assert (params != None and self.cnmpatch != None) - logging.info('we need the parameters in order to save anything\n') + logger.info('Parameters must be set in order to save anything\n') # actions on the sparse matrix cnm = self.cnmpatch.__dict__ cnmpatch = deletesparse(cnm) @@ -227,7 +228,7 @@ def save_with_compare(self, istruth=False, params=None, dview=None, Cn=None): if os.path.exists(file_path): os.remove(file_path) else: - logging.debug("nothing to remove\n") + logger.debug("nothing to remove\n") np.savez_compressed(file_path, information=information, A_full=self.comparison['cnmf_full_frame']['ourdata'][0], @@ -235,7 +236,7 @@ def save_with_compare(self, istruth=False, params=None, dview=None, Cn=None): A_patch=self.comparison['cnmf_on_patch']['ourdata'][0], C_patch=self.comparison['cnmf_on_patch']['ourdata'][1], rig_shifts=self.comparison['rig_shifts']['ourdata']) - logging.info('we now have ground truth\n') + logger.info('we now have ground truth\n') return else: # if not we create a comparison first @@ -250,7 +251,7 @@ def save_with_compare(self, istruth=False, params=None, dview=None, Cn=None): # if we cannot manage to open it or it doesn't exist: except (IOError, OSError): # we save but we explain why there were a problem - logging.warning('we were not able to read the file ' + str(file_path) + ' to compare it\n') + logger.warning(f'We were not able to read the file {file_path} to compare it\n') file_path = os.path.join(caiman_datadir(), "testdata", "NC" + dt + ".npz") np.savez_compressed(file_path, information=information, @@ -273,26 +274,26 @@ def save_with_compare(self, istruth=False, params=None, dview=None, Cn=None): information.update({'differences': {'proc': False, 'params_movie': False, 'params_cnm': False}}) if data['processor'] != information['processor']: - logging.info("You don't have the same processor as was used to generate the ground truth. The processing time can vary.\n" + + logger.info("You don't have the same processor as was used to generate the ground truth. The processing time can vary.\n" + "For time comparison, Create your own groundtruth standard for future testing.\n" + f"Compare: {data['processor']} to {information['processor']}\n") information['differences']['proc'] = True if data['params'] != information['params']: - logging.warning("You are not using the same movie parameters. Results will not be comparable.") - logging.warning('You must use the same parameters as the groundtruth.\n' + + logger.warning("You are not using the same movie parameters. Results will not be comparable.") + logger.warning('You must use the same parameters as the groundtruth.\n' + 'examine the groundtruth parameters with the see() method\n') information['differences']['params_movie'] = True # We must cleanup some fields to permit an accurate comparison if not normalised_compare_cnmpatches(data['cnmpatch'], cnmpatch): if data['cnmpatch'].keys() != cnmpatch.keys(): - logging.error( + logger.error( 'DIFFERENCES IN THE FIELDS OF CNMF' ) # TODO: Now that we have deeply nested data structures, find a module that gives you tight differences. diffkeys = [k for k in data['cnmpatch'] if data['cnmpatch'][k] != cnmpatch[k]] for k in diffkeys: - logging.info(f"{k}:{data['cnmpatch'][k]}->{cnmpatch[k]}") + logger.info(f"{k}:{data['cnmpatch'][k]}->{cnmpatch[k]}") - logging.warning('You are not using the same parameters in your cnmf on patches initialization\n') + logger.warning('You are not using the same parameters in your cnmf on patches initialization\n') information['differences']['params_cnm'] = True # for rigid @@ -373,7 +374,7 @@ def see(filename=None): dr = os.path.join(caiman_datadir(), "testdata", "groundtruth.npz") else: dr = os.path.join(caiman_datadir(), "testdata", filename, filename + ".npz") - logging.debug("Loading GT file " + str(dr)) + logger.debug("Loading GT file " + str(dr)) with np.load(dr) as dt: print('Info :\n') see_it(dt) @@ -403,7 +404,7 @@ def deletesparse(cnm): val = deletesparse(val) if not isinstance(val, scipy.sparse.coo.coo_matrix) and not isinstance(val, np.ndarray) \ and not isinstance(val, scipy.sparse.csc.csc_matrix) and not keys == 'dview': - logging.debug(f"type of val is {type(val)}") + logger.debug(f"type of val is {type(val)}") cnm[keys] = val else: @@ -483,7 +484,6 @@ def cnmf(Cn, A_gt, A_test, C_gt, C_test, dims_gt, dims_test, dview=None, sensiti def plotrig(init, curr, timer, sensitivity): - diff = np.linalg.norm(np.asarray(init) - np.asarray(curr)) / np.linalg.norm(init) isdiff = diff > sensitivity info = {'isdifferent': int(isdiff), 'diff_data': diff, 'diff_timing': timer} @@ -491,20 +491,6 @@ def plotrig(init, curr, timer, sensitivity): init = init.transpose([1, 0]) xc = np.arange(curr.shape[1]) xi = np.arange(init.shape[1]) - #try: - # pl.figure() - # pl.subplot(1, 2, 1) - # pl.plot(xc, curr[0], 'r', xi, init[0], 'b') - # pl.legend(['x shifts curr', 'x shifts init']) - # pl.xlabel('frames') - # pl.ylabel('pixels') - # pl.subplot(1, 2, 2) - # pl.plot(xc, curr[1], 'r', xi, init[1], 'b') - # pl.legend(['yshifts curr', 'y shifts init']) - # pl.xlabel('frames') - # pl.ylabel('pixels') - #except: - # logging.warning("not able to plot") return info diff --git a/caiman/tests/comparison_general.py b/caiman/tests/comparison_general.py index da0538de8..5361dad28 100644 --- a/caiman/tests/comparison_general.py +++ b/caiman/tests/comparison_general.py @@ -6,15 +6,7 @@ See Also ------------ caiman/tests/comparison/comparison.py - - """ -#%% -#\package None -#\version 1.0 -#\copyright GNU General Public License v2.0 -#\date Created on june 2017 -#\author: Jremie KALFON import copy import cv2 @@ -49,7 +41,8 @@ # You can log to a file using the filename parameter, or make the output more or less # verbose by setting level to logging.DEBUG, logging.INFO, logging.WARNING, or logging.ERROR -logging.getLogger("caiman").setLevel(logging.DEBUG) +logger = logging.getLogger("caiman") +logger.setLevel(logging.DEBUG) # GLOBAL VAR params_movie = { @@ -201,7 +194,7 @@ def test_general(): if len(name_new) > 1: fname_new = cm.save_memmap_join(name_new, base_name='Yr', n_chunks=params_movie['n_chunks'], dview=None) else: - logging.warning('One file only, not saving!') + logger.warning('One file only, not saving!') fname_new = name_new[0] Yr, dims, T = cm.load_memmap(fname_new) @@ -256,7 +249,7 @@ def test_general(): b_tot = cnm.estimates.b f_tot = cnm.estimates.f # DISCARDING - logging.info(('Number of components:' + str(A_tot.shape[-1]))) + logger.info(('Number of components:' + str(A_tot.shape[-1]))) final_frate = params_movie['final_frate'] # threshold on space consistency r_values_min = params_movie['r_values_min_patch'] @@ -332,29 +325,29 @@ def test_general(): for log_file in log_files: os.remove(log_file) except: - logging.warning('Cannot remove log files') + logger.warning('Cannot remove log files') ############ assertions ################## pb = False if (comp.information['differences']['params_movie']): - logging.error( - "you need to set the same movie parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" + logger.error( + "You must set the same movie parameters as the ground truth to have a real comparison (use the comp.see() function to explore it)" ) pb = True if (comp.information['differences']['params_cnm']): - logging.warning( - "you need to set the same cnmf parameters than the ground truth to have a real comparison (use the comp.see() function to explore it)" + logger.warning( + "You must set the same cnmf parameters as the ground truth to have a real comparison (use the comp.see() function to explore it)" ) # pb = True if (comp.information['diff']['rig']['isdifferent']): - logging.error("the rigid shifts are different from the groundtruth ") + logger.error("the rigid shifts are different from the groundtruth ") pb = True if (comp.information['diff']['cnmpatch']['isdifferent']): - logging.error("the cnmf on patch produces different results than the groundtruth ") + logger.error("the cnmf on patch produces different results than the groundtruth ") pb = True if (comp.information['diff']['cnmfull']['isdifferent']): - logging.error("the cnmf full frame produces different results than the groundtruth ") + logger.error("the cnmf full frame produces different results than the groundtruth ") pb = True assert (not pb) diff --git a/caiman/tests/test_deconvolution.py b/caiman/tests/test_deconvolution.py index 46694a0fc..a34eaa086 100644 --- a/caiman/tests/test_deconvolution.py +++ b/caiman/tests/test_deconvolution.py @@ -11,7 +11,8 @@ # You can log to a file using the filename parameter, or make the output more or less # verbose by setting level to logging.DEBUG, logging.INFO, logging.WARNING, or logging.ERROR -logging.getLogger("caiman").setLevel(logging.DEBUG) +logger = logging.getLogger("caiman") +logger.setLevel(logging.DEBUG) def gen_data(g=[.95], sn=.2, T=1000, framerate=30, firerate=.5, b=10, N=1, seed=0): """ @@ -67,7 +68,7 @@ def foo(method, p): res = constrained_foopsi(y, g=g, sn=sn, p=p, method=method) npt.assert_allclose(np.corrcoef(res[0], c)[0, 1], 1, [.01, .1][i]) npt.assert_allclose(np.corrcoef(res[-2], s)[0, 1], 1, [.03, .3][i]) - logging.debug(['\n', ''][p - 1] + ' %5s AR%d %.4fs' % (method, p, time() - t)) + logger.debug(['\n', ''][p - 1] + ' %5s AR%d %.4fs' % (method, p, time() - t)) def test_oasis(): diff --git a/demos/general/demo_OnACID.py b/demos/general/demo_OnACID.py index ca56b41d1..39418ef7d 100755 --- a/demos/general/demo_OnACID.py +++ b/demos/general/demo_OnACID.py @@ -57,7 +57,7 @@ def main(): cnm.fit_online() # plot contours - logging.info(f"Number of components: {cnm.estimates.A.shape[-1]}") + logger.info(f"Number of components: {cnm.estimates.A.shape[-1]}") Cn = cm.load(fnames[0], subindices=slice(0,500)).local_correlations(swap_dim=False) cnm.estimates.plot_contours(img=Cn) diff --git a/demos/general/demo_OnACID_mesoscope.py b/demos/general/demo_OnACID_mesoscope.py index a3afe5d57..4e1fe86f6 100755 --- a/demos/general/demo_OnACID_mesoscope.py +++ b/demos/general/demo_OnACID_mesoscope.py @@ -69,7 +69,7 @@ def main(): cnm.fit_online() # plot contours (this may take time) - logging.info(f"Number of components: {cnm.estimates.A.shape[-1]}") + logger.info(f"Number of components: {cnm.estimates.A.shape[-1]}") images = cm.load(fnames) Cn = images.local_correlations(swap_dim=False, frames_per_chunk=500) cnm.estimates.plot_contours(img=Cn, display_numbers=False) diff --git a/demos/notebooks/demo_OnACID_mesoscope.ipynb b/demos/notebooks/demo_OnACID_mesoscope.ipynb index b6d2407ab..cca9bb663 100644 --- a/demos/notebooks/demo_OnACID_mesoscope.ipynb +++ b/demos/notebooks/demo_OnACID_mesoscope.ipynb @@ -175,7 +175,7 @@ "metadata": {}, "outputs": [], "source": [ - "logging.info('Number of components: ' + str(cnm.estimates.A.shape[-1]))\n", + "logger.info('Number of components: ' + str(cnm.estimates.A.shape[-1]))\n", "Cn = cm.load(fnames[0], subindices=slice(0,500)).local_correlations(swap_dim=False)\n", "cnm.estimates.plot_contours(img=Cn)" ]