added SLURM options to config file and process-level resource config…

workflows/Nextflow/modules/countFastq/countFastq.nf

@@ -1,6 +1,7 @@
 //creates fastqCount.txt as preparation for downstream processes
 process countFastq {
     label "countFastq"
+    label "small"
 
     input:
     val settings
@@ -30,6 +31,7 @@ process countFastq {
 //gets average read length from fastqCount.txt
 process avgLen {
     label "countFastq"
+    label "small"
 
     input:
     path countFastq

workflows/Nextflow/modules/hostRemoval/hostRemoval.nf

@@ -5,6 +5,8 @@
 //clean input FASTQ files of reads that map to provided host references, one process per given host
 process hostRemoval {
     label "hostRemoval"
+    label "medium"
+
     publishDir(
         path: "${settings["outDir"]}/HostRemoval",
         mode: 'copy'
@@ -46,13 +48,12 @@ process hostRemoval {
     }
     minScore = ontFlag != "" ? "-T ${settings["minLen"]} " : minScore
     bwaMemOptions = "-bwaMemOptions \"${ontFlag} ${minScore}\" "
-    def cpu = settings["cpus"] != null ? "-cpu ${settings["cpus"]} " : ""
 
     """
     host_reads_removal_by_mapping.pl\
     $refFile\
     $prefix\
-    $cpu\
+    -cpu ${task.cpus} \
     -host \
     $bwaMemOptions\
     $pairedFiles\
@@ -68,6 +69,8 @@ process hostRemoval {
 //merge cleaned FASTQ files into files cleaned of ALL reads mapping to ANY provided host reference
 process collectCleanPairedReads {
     label "hostRemoval"
+    label "small"
+
     publishDir(
         path: "${settings["outDir"]}/HostRemoval",
         mode: 'copy'
@@ -94,6 +97,8 @@ process collectCleanPairedReads {
 
 process collectCleanPairedReadsOneHost {
     label "hostRemoval"
+    label "small"
+
     publishDir(
         path: "${settings["outDir"]}/HostRemoval",
         mode: 'copy'
@@ -116,6 +121,8 @@ process collectCleanPairedReadsOneHost {
 //Concatenate leftover unpaired reads that didn't map to a host reference, and remove any name duplicates (i.e., leftovers appearing in multiple cleanings)
 process collectCleanSingleReads {
     label "hostRemoval"
+    label "small"
+
     publishDir(
         path: "${settings["outDir"]}/HostRemoval",
         mode: 'copy'
@@ -137,6 +144,8 @@ process collectCleanSingleReads {
 
 process hostRemovalStats {
     label "hostRemoval"
+    label "small"
+
     publishDir "${settings["outDir"]}/HostRemoval", mode: 'copy'
 
     input:

workflows/Nextflow/modules/runAssembly/runAssembly.nf

@@ -7,6 +7,7 @@
 //main process for assembly with IDBA
 process idbaUD {
     label "assembly"
+    label "medium"
     publishDir (
     path:"${settings["outDir"]}/AssemblyBasedAnalysis",
     mode: 'copy',
@@ -64,7 +65,7 @@ process idbaUD {
 
     memLimit = settings["memLimit"] != null ? "ulimit -v ${settings["memLimit"]} 2>/dev/null;" : ""
     """
-    ${memLimit}idba_ud --pre_correction -o . --num_threads ${settings["cpus"]}\
+    ${memLimit}idba_ud --pre_correction -o . --num_threads ${task.cpus}\
     $runFlag\
     $longReadsFile\
     $maxK_option\
@@ -80,6 +81,7 @@ process idbaUD {
 //prep for idba
 process idbaExtractLong {
     label "assembly"
+    label "small"
 
     input:
     path paired
@@ -104,6 +106,8 @@ process idbaExtractLong {
 //prep for idba
 process idbaPrepReads {
     label "assembly"
+    label "small"
+
     input:
     path paired
     path unpaired
@@ -127,6 +131,7 @@ process idbaPrepReads {
 //assemble using spades
 process spades {
     label "assembly"
+    label "medium"
 
     publishDir (
     path: "${settings["outDir"]}/AssemblyBasedAnalysis", 
@@ -191,7 +196,7 @@ process spades {
     def memLimit = settings["memLimit"] != null ? "-m ${settings["memLimit"]}" : ""
 
     """
-    spades.py -o . -t ${settings["cpus"]}\
+    spades.py -o . -t ${task.cpus}\
     $paired\
     $meta_flag\
     $sc_flag\
@@ -211,8 +216,9 @@ process spades {
 
 //assemble using megahit
 process megahit {
-
     label "assembly"
+    label "medium"
+
     publishDir(
     path: "${settings["outDir"]}/AssemblyBasedAnalysis", 
     mode: 'copy',
@@ -254,7 +260,7 @@ process megahit {
     def megahit_preset = settings["megahit"]["preset"] != null ? "--presets ${settings["megahit"]["preset"]} " : ""
 
     """
-    megahit -o ./megahit -t ${settings["cpus"]}\
+    megahit -o ./megahit -t ${task.cpus}\
     $megahit_preset\
     $paired\
     $unpaired\
@@ -271,6 +277,8 @@ process megahit {
 //assembly using unicycler
 process unicycler {
     label "assembly"
+    label "medium"
+
     publishDir (
         path: "${settings["outDir"]}/AssemblyBasedAnalysis", 
         mode: 'copy',
@@ -316,7 +324,7 @@ process unicycler {
     """
     export _JAVA_OPTIONS='-Xmx20G'; export TERM='xterm';
 
-    unicycler -t ${settings["cpus"]} -o .\
+    unicycler -t ${task.cpus} -o .\
     $paired\
     $filt_lr\
     $bridge 2>&1 1>/dev/null
@@ -327,6 +335,7 @@ process unicycler {
 //filter long reads for unicycler
 process unicyclerPrep {
     label "assembly"
+    label "small"
 
     input:
     val settings
@@ -349,6 +358,7 @@ process unicyclerPrep {
 //assembly using lrasm
 process lrasm {
     label "assembly"
+    label "medium"
 
     publishDir (
         path: "${settings["outDir"]}/AssemblyBasedAnalysis", 
@@ -414,7 +424,7 @@ process lrasm {
     def flyeOpt = settings["lrasm"]["algorithm"] == "metaflye" ? "--fo '--meta' ": ""
 
     """
-    lrasm -o . -t ${settings["cpus"]} \
+    lrasm -o . -t ${task.cpus} \
     $preset\
     $consensus\
     $errorCorrection\
@@ -427,6 +437,8 @@ process lrasm {
 
 process renameFilterFasta {
     label "assembly"
+    label "small"
+
     publishDir(
         path: "${settings["outDir"]}/AssemblyBasedAnalysis",
         mode: 'copy'
@@ -458,6 +470,8 @@ process renameFilterFasta {
 
 process bestIncompleteAssembly {
     label "assembly"
+    label "small"
+
     input:
 
     val x

workflows/Nextflow/modules/runFaQCs/runFaQCs.nf

@@ -3,6 +3,8 @@
 //double-checks that any provided adapter file is in FASTA format
 process adapterFileCheck {
     label "qc"
+    label "small"
+
     input:
     path adapterFile
 
@@ -18,6 +20,8 @@ process adapterFileCheck {
 //main QC process. puts parameters together and runs FaQCs.
 process qc {
     label "qc"
+    label "medium"
+
     publishDir(
         path: "${settings["outDir"]}/QcReads",
         mode: 'copy'
@@ -61,7 +65,7 @@ process qc {
     $qcSoftware $inputArg \
     -q ${settings["trimQual"]} --min_L $min --avg_q ${settings["avgQual"]} \
     -n ${settings["numN"]} --lc ${settings["filtLC"]} --5end ${settings["trim5end"]} --3end ${settings["trim3end"]} \
-    --split_size 1000000 -d . -t ${settings["cpus"]} \
+    --split_size 1000000 -d . -t ${task.cpus} \
     $polyA \
     $adapterArg \
     $phiX

workflows/Nextflow/modules/runReadsToContig/runReadsToContig.nf

@@ -2,6 +2,8 @@
 
 process validationAlignment {
     label 'r2c'
+    label "medium"
+
     publishDir(
         path: "${settings["outDir"]}/AssemblyBasedAnalysis/readsMappingToContig",
         mode: 'copy'
@@ -25,7 +27,6 @@ process validationAlignment {
     def paired = paired.name != "NO_FILE" ? "-p \'${paired[0]} ${paired[1]}\' " : ""
     def unpaired = unpaired.name != "NO_FILE2" ? "-u $unpaired " : ""
     def cutoff = settings["useAssembledContigs"] ? "-c 0 " : "-c 0.1 "
-    def cpu = settings["cpus"] != null ? "-cpu ${settings["cpus"]} " : ""
     def max_clip = settings["r2g_max_clip"] != null ? "-max_clip ${settings["r2g_max_clip"]} " : ""
 
 
@@ -67,7 +68,7 @@ process validationAlignment {
     """
     runReadsToContig.pl \
     $cutoff\
-    $cpu\
+    -cpu ${task.cpus}\
     $paired\
     $unpaired\
     -d . -pre $outPrefix\
@@ -83,6 +84,8 @@ process validationAlignment {
 
 process makeCoverageTable {
     label 'r2c'
+    label "small"
+
     publishDir(
         path: "${settings["outDir"]}/AssemblyBasedAnalysis/readsMappingToContig",
         mode: 'copy',
@@ -116,6 +119,8 @@ process makeCoverageTable {
 
 process extractUnmapped {
     label 'r2c'
+    label "small"
+
     publishDir(
         path:"${settings["outDir"]}/AssemblyBasedAnalysis/readsMappingToContig/",
         mode: 'copy',

workflows/Nextflow/modules/sra2fastq/sra2fastq.nf

@@ -6,6 +6,8 @@
 
 process sraDownload {
     label "sra2fastq"
+    label "small"
+
     tag "$accession"
     publishDir "${settings["outDir"]}/SRA_Download", mode: 'copy'
 

workflows/Nextflow/nextflow.config

@@ -106,16 +106,43 @@ params {
 
 }
 
-//container settings
-apptainer {
-    enabled = true
-    pullTimeout = "1 hour"
-    runOptions = "--compat"
-}
 
+executor {
+    //which executor to use.
+    //for SLURM job scheduling, replace 'local' with 'slurm'
+    name = "local"
+
+    //controls name of jobs submitted to cluster executor
+    jobName = {"job_name_$task.name"}
 
-//which containers to use for which processes
+    //submission rate limit: needed for sra2fastq to operate correctly
+    submitRateLimit = '1/5sec'
+}
 process {
+
+    //options to provide for all processes submitted as cluster jobs (e.g. "--account xx")
+    //do not use to specify cpu or memory requirements, those are handled below
+    clusterOptions = ""
+
+    //resource requirements for processes
+    withLabel: 'small' {
+        cpus = 4
+        memory = '8 GB'
+    }
+    withLabel: 'medium' {
+        cpus = 8
+        memory= '16 GB'
+    }
+    withLabel: 'large' {
+        cpus = 16
+        memory = '32 GB'
+    }
+    withLabel: 'largest' {
+        cpus = 16
+        memory = '64 GB'
+    }
+
+    //which containers to use for which processes
     withLabel: 'sra2fastq' {
         container = 'kaijli/sra2fastq:1.6.3'
     }
@@ -134,13 +161,17 @@ process {
     withLabel: 'r2c' {
         container = 'apwat/run_r2c:1.6'
     }
+
 }
 
-//submission rate limit: needed for sra2fastq to operate correctly
-executor {
-    submitRateLimit = '1/5sec'
+//container runtime settings
+apptainer {
+    enabled = true
+    pullTimeout = "1 hour"
+    runOptions = "--compat"
 }
 
+
 //cleanup
 cleanup = false