From 97451bac06990fc3447e320dd6d34168982638ae Mon Sep 17 00:00:00 2001 From: Pat Gunn Date: Fri, 19 Apr 2024 15:17:14 -0400 Subject: [PATCH] Adjust gating, path handling to try harder not to drop temporary files anywhere but caiman_data/temp/ --- caiman/motion_correction.py | 9 ++------- caiman/paths.py | 11 ++++++----- caiman/source_extraction/cnmf/online_cnmf.py | 6 ++++-- demos/general/demo_pipeline_NWB.py | 4 ++-- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/caiman/motion_correction.py b/caiman/motion_correction.py index d2e9b6bef..04283c527 100644 --- a/caiman/motion_correction.py +++ b/caiman/motion_correction.py @@ -167,9 +167,9 @@ def __init__(self, fname, min_mov=None, dview=None, max_shifts=(6, 6), niter_rig """ if 'ndarray' in str(type(fname)) or isinstance(fname, caiman.base.movies.movie): - mc_tempfile = os.path.join(caiman.paths.get_tempdir(), 'tmp_mov_mot_corr.hdf5') + mc_tempfile = caiman.paths.fn_relocated('tmp_mov_mot_corr.hdf5') if os.path.isfile(mc_tempfile): - os.remove(mc_tempfile) # Eventually get_tempdir() will keep jobs separate and make this safer + os.remove(mc_tempfile) logging.info(f"Creating file for motion correction: {mc_tempfile}") caiman.movie(fname).save(mc_tempfile) fname = [mc_tempfile] @@ -3123,12 +3123,7 @@ def motion_correction_piecewise(fname, splits, strides, overlaps, add_to_movie=0 if base_name is None: base_name = os.path.splitext(os.path.split(fname)[1])[0] base_name = caiman.paths.fn_relocated(base_name) - fname_tot:Optional[str] = caiman.paths.memmap_frames_filename(base_name, dims, T, order) - if isinstance(fname, tuple): - fname_tot = os.path.join(os.path.split(fname[0])[0], fname_tot) - else: - fname_tot = os.path.join(os.path.split(fname)[0], fname_tot) np.memmap(fname_tot, mode='w+', dtype=np.float32, shape=caiman.mmapping.prepare_shape(shape_mov), order=order) diff --git a/caiman/paths.py b/caiman/paths.py index ae5edbbe7..05abfb336 100644 --- a/caiman/paths.py +++ b/caiman/paths.py @@ -45,7 +45,7 @@ def get_tempdir() -> str: os.makedirs(temp_under_data) return temp_under_data -def fn_relocated(fn:str) -> str: +def fn_relocated(fn:str, force_temp:bool=False) -> str: """ If the provided filename does not contain any path elements, this returns what would be its absolute pathname as located in get_tempdir(). Otherwise it just returns what it is passed. @@ -53,10 +53,10 @@ def fn_relocated(fn:str) -> str: but if all they think about is filenames, they go under CaImAn's notion of its temporary dir. This is under the principle of "sensible defaults, but users can override them". """ - if not 'CAIMAN_NEW_TEMPFILE' in os.environ: # XXX We will ungate this in a future version of caiman - return fn - if str(os.path.basename(fn)) == str(fn): # No path stuff + if os.path.split(fn)[0] == '': # No path stuff return os.path.join(get_tempdir(), fn) + elif force_temp: + return os.path.join(get_tempdir(), os.path.split(fn)[1]) else: return fn @@ -124,4 +124,5 @@ def generate_fname_tot(base_name:str, dims:list[int], order:str) -> str: d1, d2, d3 = dims[0], dims[1], dims[2] ret = '_'.join([base_name, 'd1', str(d1), 'd2', str(d2), 'd3', str(d3), 'order', order]) ret = re.sub(r'(_)+', '_', ret) # Turn repeated underscores into just one - return ret + return fn_relocated(ret, force_temp=True) + diff --git a/caiman/source_extraction/cnmf/online_cnmf.py b/caiman/source_extraction/cnmf/online_cnmf.py index fefb1f100..2a1aede6c 100644 --- a/caiman/source_extraction/cnmf/online_cnmf.py +++ b/caiman/source_extraction/cnmf/online_cnmf.py @@ -20,6 +20,7 @@ from math import sqrt from multiprocessing import cpu_count import numpy as np +import os from scipy.ndimage import percentile_filter from scipy.sparse import coo_matrix, csc_matrix, spdiags, hstack from scipy.stats import norm @@ -958,8 +959,9 @@ def initialize_online(self, model_LN=None, T=None): self.estimates = cnm.estimates else: - Y.save(caiman.paths.fn_relocated('init_file.hdf5')) - f_new = caiman.mmapping.save_memmap(['init_file.hdf5'], base_name='Yr', order='C', + temp_init_file = os.path.join(caiman.paths.get_tempdir(), 'init_file.hdf5') + Y.save(temp_init_file) + f_new = caiman.mmapping.save_memmap([temp_init_file], base_name='Yr', order='C', slices=[slice(0, opts['init_batch']), None, None]) Yrm, dims_, T_ = caiman.mmapping.load_memmap(f_new) diff --git a/demos/general/demo_pipeline_NWB.py b/demos/general/demo_pipeline_NWB.py index 22e66f139..956f632b1 100755 --- a/demos/general/demo_pipeline_NWB.py +++ b/demos/general/demo_pipeline_NWB.py @@ -25,7 +25,7 @@ import caiman from caiman.motion_correction import MotionCorrect -from caiman.paths import caiman_datadir +from caiman.paths import caiman_datadir, get_tempdir from caiman.source_extraction.cnmf import cnmf as cnmf from caiman.source_extraction.cnmf import params as params from caiman.utils.utils import download_demo @@ -58,7 +58,7 @@ def main(): # The convert target is the original fn, with the extension swapped to nwb. fr = float(15) # imaging rate in frames per second pre_convert = download_demo('Sue_2x_3000_40_-46.tif') - convert_target = os.path.join(caiman_datadir(), 'Sue_2x_3000_40_-46.nwb') + convert_target = os.path.join(get_tempdir(), 'Sue_2x_3000_40_-46.nwb') orig_movie = caiman.load(pre_convert, fr=fr) # save file in NWB format with various additional info