Merge pull request #97 from ENCODE-DCC/dev

v1.4.1

caper/__init__.py

@@ -2,4 +2,4 @@
 from .caper_runner import CaperRunner
 
 __all__ = ['CaperClient', 'CaperClientSubmit', 'CaperRunner']
-__version__ = '1.4.0'
+__version__ = '1.4.1'

caper/cli.py

@@ -635,7 +635,7 @@ def subcmd_cleanup(caper_client, args):
     """Cleanup outputs of a workflow.
     """
     cm = get_single_cromwell_metadata_obj(caper_client, args, 'cleanup')
-    cm.cleanup(dry_run=not args.delete, num_threads=args.num_threads)
+    cm.cleanup(dry_run=not args.delete, num_threads=args.num_threads, no_lock=True)
     if not args.delete:
         logger.warning(
             'Use --delete to DELETE ALL OUTPUTS of this workflow. '

caper/cromwell_metadata.py

@@ -386,7 +386,9 @@ def add_to_input_files_if_valid(file):
         json_str = json.dumps(result, default=convert_type_np_to_py)
         return json.loads(json_str)
 
-    def cleanup(self, dry_run=False, num_threads=URIBase.DEFAULT_NUM_THREADS):
+    def cleanup(
+        self, dry_run=False, num_threads=URIBase.DEFAULT_NUM_THREADS, no_lock=False
+    ):
         """Cleans up workflow's root output directory.
 
         Args:
@@ -396,6 +398,8 @@ def cleanup(self, dry_run=False, num_threads=URIBase.DEFAULT_NUM_THREADS):
                 For outputs on cloud buckets only.
                 Number of threads for deleting individual outputs on cloud buckets in parallel.
                 Generates one client per thread. This works like `gsutil -m rm -rf`.
+            no_lock:
+                No file locking.
         """
         root = self._metadata.get('workflowRoot')
         if root is None:
@@ -407,6 +411,8 @@ def cleanup(self, dry_run=False, num_threads=URIBase.DEFAULT_NUM_THREADS):
 
         if AbsPath(root).is_valid:
             # num_threads is not available for AbsPath().rmdir()
-            AbsPath(root).rmdir(dry_run=dry_run)
+            AbsPath(root).rmdir(dry_run=dry_run, no_lock=no_lock)
         else:
-            AutoURI(root).rmdir(dry_run=dry_run, num_threads=num_threads)
+            AutoURI(root).rmdir(
+                dry_run=dry_run, no_lock=no_lock, num_threads=num_threads
+            )

caper/cromwell_workflow_monitor.py

@@ -296,6 +296,13 @@ def _update_metadata(self, workflow_id):
         """
         if not self._is_server or not self._auto_update_metadata:
             return
+        if workflow_id in self._subworkflows and self._embed_subworkflow:
+            logger.debug(
+                'Skipped writing metadata JSON file of subworkflow {wf_id}'.format(
+                    wf_id=workflow_id
+                )
+            )
+            return
         for trial in range(CromwellWorkflowMonitor.MAX_RETRY_UPDATE_METADATA + 1):
             try:
                 time.sleep(CromwellWorkflowMonitor.INTERVAL_RETRY_UPDATE_METADATA)

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='caper',
-    version='1.4.0',
+    version='1.4.1',
     python_requires='>=3.6',
     scripts=[
         'bin/caper',
@@ -29,7 +29,7 @@
         'pyhocon>=0.3.53',
         'requests>=2.20',
         'pyopenssl',
-        'autouri>=0.2.2',
+        'autouri>=0.2.3',
         'miniwdl>=0.7.0',
         'humanfriendly',
         'numpy>=1.8.2',