cli demos: make amount of parallelism in the backend a CLI parameter

demos/general/demo_OnACID_mesoscope.py

@@ -126,9 +126,7 @@ def main():
     plt.ylabel('Processing time [ms]')
 
     # Prepare result visualisations (might take time)
-    c, dview, n_processes = \
-        cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=None,
-                                 single_thread=False)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
     if opts.online['motion_correct']:
         shifts = cnm.estimates.shifts[-cnm.estimates.C.shape[-1]:]
         if not opts.motion['pw_rigid']:
@@ -171,6 +169,7 @@ def handle_args():
     parser = argparse.ArgumentParser(description="Full OnACID Caiman demo")
     parser.add_argument("--no_play",    action="store_true", help="Do not display results")
     parser.add_argument("--cluster_backend", default="multiprocessing", help="Specify multiprocessing, ipyparallel, or single to pick an engine")
+    parser.add_argument("--cluster_nproc", type=int, default=None, help="Override automatic selection of number of workers to use")
     parser.add_argument("--input", action="append", help="File(s) to work on, provide multiple times for more files")
     parser.add_argument("--logfile",    help="If specified, log to the named file")
     return parser.parse_args()

demos/general/demo_caiman_basic.py

@@ -92,7 +92,7 @@ def main():
     opts = params.CNMFParams(params_dict=params_dict)
 
     # start a cluster for parallel processing
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
 
     # Run CaImAn Batch (CNMF)
@@ -170,6 +170,7 @@ def handle_args():
     parser.add_argument("--no_patches", action="store_true", help="Do not use patches")
     parser.add_argument("--no_play",    action="store_true", help="Do not display results")
     parser.add_argument("--cluster_backend", default="multiprocessing", help="Specify multiprocessing, ipyparallel, or single to pick an engine")
+    parser.add_argument("--cluster_nproc", type=int, default=None, help="Override automatic selection of number of workers to use")
     parser.add_argument("--input", action="append", help="File(s) to work on, provide multiple times for more files")
     parser.add_argument("--logfile",    help="If specified, log to the named file")
     return parser.parse_args()

demos/general/demo_pipeline.py

@@ -102,7 +102,7 @@ def main():
         moviehandle.play(q_max=99.5, fr=60, magnification=2)
 
     # start a cluster for parallel processing
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
     # Motion Correction
     # first we create a motion correction object with the specified parameters
@@ -138,7 +138,7 @@ def main():
 
     # restart cluster to clean up memory
     cm.stop_server(dview=dview)
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
     #  Parameters for source extraction and deconvolution
     p = 1                    # order of the autoregressive system
@@ -264,6 +264,7 @@ def handle_args():
     parser.add_argument("--keep_logs",  action="store_true", help="Keep temporary logfiles")
     parser.add_argument("--no_play",    action="store_true", help="Do not display results")
     parser.add_argument("--cluster_backend", default="multiprocessing", help="Specify multiprocessing, ipyparallel, or single to pick an engine")
+    parser.add_argument("--cluster_nproc", type=int, default=None, help="Override automatic selection of number of workers to use")
     parser.add_argument("--input", action="append", help="File(s) to work on, provide multiple times for more files")
     parser.add_argument("--logfile",    help="If specified, log to the named file")
     return parser.parse_args()

demos/general/demo_pipeline_NWB.py

@@ -161,7 +161,7 @@ def main():
         moviehandle.play(q_max=99.5, fr=60, magnification=2)
 
     # start a cluster for parallel processing
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
     # Motion Correction
     # first we create a motion correction object with the specified parameters
@@ -197,8 +197,7 @@ def main():
 
     # restart cluster to clean up memory
     cm.stop_server(dview=dview)
-    c, dview, n_processes = cm.cluster.setup_cluster(
-        backend=cfg.cluster_backend, n_processes=None, single_thread=False)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
     #  parameters for source extraction and deconvolution
     p = 1                    # order of the autoregressive system
@@ -331,6 +330,7 @@ def handle_args():
     parser.add_argument("--keep_logs",  action="store_true", help="Keep temporary logfiles")
     parser.add_argument("--no_play",    action="store_true", help="Do not display results")
     parser.add_argument("--cluster_backend", default="multiprocessing", help="Specify multiprocessing, ipyparallel, or single to pick an engine")
+    parser.add_argument("--cluster_nproc", type=int, default=None, help="Override automatic selection of number of workers to use")
     parser.add_argument("--input", action="append", help="File(s) to work on, provide multiple times for more files")
     parser.add_argument("--logfile",    help="If specified, log to the named file")
     return parser.parse_args()

demos/general/demo_pipeline_cnmfE.py

@@ -83,7 +83,7 @@ def main():
 
     opts = params.CNMFParams(params_dict=params_dict)
 
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
     # Motion Correction
     #  The pw_rigid flag set above, determines where to use rigid or pw-rigid
     #  motion correction
@@ -230,6 +230,7 @@ def handle_args():
     parser.add_argument("--keep_logs",  action="store_true", help="Keep temporary logfiles")
     parser.add_argument("--no_play",    action="store_true", help="Do not display results")
     parser.add_argument("--cluster_backend", default="multiprocessing", help="Specify multiprocessing, ipyparallel, or single to pick an engine")
+    parser.add_argument("--cluster_nproc", type=int, default=None, help="Override automatic selection of number of workers to use")
     parser.add_argument("--input", action="append", help="File(s) to work on, provide multiple times for more files")
     parser.add_argument("--logfile",    help="If specified, log to the named file")
     return parser.parse_args()

demos/general/demo_pipeline_voltage_imaging.py

@@ -92,7 +92,7 @@ def main():
         moviehandle.play(q_max=99.5, fr=40, magnification=4)
 
     # start a cluster for parallel processing
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
     # %%% MOTION CORRECTION
     # first we create a motion correction object with the specified parameters
@@ -179,7 +179,7 @@ def main():
 
     # restart cluster to clean up memory
     cm.stop_server(dview=dview)
-    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend)
+    c, dview, n_processes = cm.cluster.setup_cluster(backend=cfg.cluster_backend, n_processes=cfg.cluster_nproc)
 
 
     # parameters for trace denoising and spike extraction
@@ -273,6 +273,7 @@ def handle_args():
     parser.add_argument("--keep_logs",  action="store_true", help="Keep temporary logfiles")
     parser.add_argument("--no_play",    action="store_true", help="Do not display results")
     parser.add_argument("--cluster_backend", default="multiprocessing", help="Specify multiprocessing, ipyparallel, or single to pick an engine")
+    parser.add_argument("--cluster_nproc", type=int, default=None, help="Override automatic selection of number of workers to use")
     parser.add_argument("--input", action="append", help="File(s) to work on, provide multiple times for more files")
     parser.add_argument("--pathinput", action="append", help="Path input file(s) to work on, provide multiple times for more files")
     parser.add_argument("--logfile",    help="If specified, log to the named file")