Adjust gating, path handling to try harder not to drop temporary file…

…s anywhere but caiman_data/temp/

caiman/motion_correction.py

@@ -167,9 +167,9 @@ def __init__(self, fname, min_mov=None, dview=None, max_shifts=(6, 6), niter_rig
 
         """
         if 'ndarray' in str(type(fname)) or isinstance(fname, caiman.base.movies.movie):
-            mc_tempfile = os.path.join(caiman.paths.get_tempdir(), 'tmp_mov_mot_corr.hdf5')
+            mc_tempfile = caiman.paths.fn_relocated('tmp_mov_mot_corr.hdf5')
             if os.path.isfile(mc_tempfile):
-                os.remove(mc_tempfile) # Eventually get_tempdir() will keep jobs separate and make this safer
+                os.remove(mc_tempfile)
             logging.info(f"Creating file for motion correction: {mc_tempfile}")
             caiman.movie(fname).save(mc_tempfile)
             fname = [mc_tempfile]
@@ -3123,12 +3123,7 @@ def motion_correction_piecewise(fname, splits, strides, overlaps, add_to_movie=0
         if base_name is None:
             base_name = os.path.splitext(os.path.split(fname)[1])[0]
         base_name = caiman.paths.fn_relocated(base_name)
-
         fname_tot:Optional[str] = caiman.paths.memmap_frames_filename(base_name, dims, T, order)
-        if isinstance(fname, tuple):
-            fname_tot = os.path.join(os.path.split(fname[0])[0], fname_tot)
-        else:
-            fname_tot = os.path.join(os.path.split(fname)[0], fname_tot)
 
         np.memmap(fname_tot, mode='w+', dtype=np.float32,
                   shape=caiman.mmapping.prepare_shape(shape_mov), order=order)

caiman/paths.py

@@ -45,18 +45,18 @@ def get_tempdir() -> str:
         os.makedirs(temp_under_data)
     return temp_under_data
 
-def fn_relocated(fn:str) -> str:
+def fn_relocated(fn:str, force_temp:bool=False) -> str:
     """ If the provided filename does not contain any path elements, this returns what would be its absolute pathname
         as located in get_tempdir(). Otherwise it just returns what it is passed.
 
         The intent behind this is to ease having functions that explicitly mention pathnames have them go where they want,
         but if all they think about is filenames, they go under CaImAn's notion of its temporary dir. This is under the
         principle of "sensible defaults, but users can override them".
     """
-    if not 'CAIMAN_NEW_TEMPFILE' in os.environ: # XXX We will ungate this in a future version of caiman
-        return fn
-    if str(os.path.basename(fn)) == str(fn): # No path stuff
+    if os.path.split(fn)[0] == '': # No path stuff
         return os.path.join(get_tempdir(), fn)
+    elif force_temp:
+        return os.path.join(get_tempdir(), os.path.split(fn)[1])
     else:
         return fn
 
@@ -124,4 +124,5 @@ def generate_fname_tot(base_name:str, dims:list[int], order:str) -> str:
         d1, d2, d3 = dims[0], dims[1], dims[2]
     ret = '_'.join([base_name, 'd1', str(d1), 'd2', str(d2), 'd3', str(d3), 'order', order])
     ret = re.sub(r'(_)+', '_', ret) # Turn repeated underscores into just one
-    return ret
+    return fn_relocated(ret, force_temp=True)
+

caiman/source_extraction/cnmf/online_cnmf.py

@@ -20,6 +20,7 @@
 from math import sqrt
 from multiprocessing import cpu_count
 import numpy as np
+import os
 from scipy.ndimage import percentile_filter
 from scipy.sparse import coo_matrix, csc_matrix, spdiags, hstack
 from scipy.stats import norm
@@ -958,8 +959,9 @@ def initialize_online(self, model_LN=None, T=None):
                 self.estimates = cnm.estimates
 
             else:
-                Y.save(caiman.paths.fn_relocated('init_file.hdf5'))
-                f_new = caiman.mmapping.save_memmap(['init_file.hdf5'], base_name='Yr', order='C',
+                temp_init_file = os.path.join(caiman.paths.get_tempdir(), 'init_file.hdf5')
+                Y.save(temp_init_file)
+                f_new = caiman.mmapping.save_memmap([temp_init_file], base_name='Yr', order='C',
                                              slices=[slice(0, opts['init_batch']), None, None])
 
                 Yrm, dims_, T_ = caiman.mmapping.load_memmap(f_new)

demos/general/demo_pipeline_NWB.py

@@ -25,7 +25,7 @@
 
 import caiman
 from caiman.motion_correction import MotionCorrect
-from caiman.paths import caiman_datadir
+from caiman.paths import caiman_datadir, get_tempdir
 from caiman.source_extraction.cnmf import cnmf as cnmf
 from caiman.source_extraction.cnmf import params as params
 from caiman.utils.utils import download_demo
@@ -58,7 +58,7 @@ def main():
         # The convert target is the original fn, with the extension swapped to nwb.
         fr = float(15)      # imaging rate in frames per second
         pre_convert    = download_demo('Sue_2x_3000_40_-46.tif')
-        convert_target = os.path.join(caiman_datadir(), 'Sue_2x_3000_40_-46.nwb')
+        convert_target = os.path.join(get_tempdir(), 'Sue_2x_3000_40_-46.nwb')
 
         orig_movie = caiman.load(pre_convert, fr=fr)
         # save file in NWB format with various additional info