update retry strategy to allow 2 retries then ignore; update params.m…

…ax_memory to allow for up to 220GB (>3x72.GB) with 3rd process_high retry
tansey-lab · Jul 25, 2024 · c949564 · c949564
1 parent b2b7783
commit c949564
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/nextflow/main.nf b/nextflow/main.nf
@@ -9,7 +9,8 @@
 ----------------------------------------------------------------------------------------
 */
 
-params.max_memory = "100.GB"
+// allow 2 retries; process_high: 72.GB * task.attempt = 216.GB
+params.max_memory = "220.GB"
 params.max_cpus = 12
 
 // sub-directories - only FastQC needs additional sub-dir

diff --git a/nextflow/nextflow.config b/nextflow/nextflow.config
@@ -26,8 +26,10 @@ process {
     memory = { check_max( 6.GB * task.attempt, 'memory' ) }
     time   = { check_max( 4.h  * task.attempt, 'time'   ) }
 
-    errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
-    maxRetries    = 1
+    // https://github.com/nextflow-io/nextflow/issues/563
+    errorStrategy = {  ( task.attempt <= process.maxRetries ) ? 'retry' : 'ignore' }
+    // errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
+    maxRetries    = 2
     maxErrors     = '-1'
 
     withLabel:process_single {