diff --git a/asm_annot.nf b/asm_annot.nf index 051bfba..980963f 100644 --- a/asm_annot.nf +++ b/asm_annot.nf @@ -39,7 +39,7 @@ Channel // run_fastq and run_multiqc are exactly the same as qc_track process run_fastqc { - publishDir "${params.out_dir}/fastqc", mode: 'copy' + publishDir "${params.out_dir}/fastqc", mode: "${params.savemode}" tag { pair_id } label 'one' @@ -50,13 +50,14 @@ process run_fastqc { file "$pair_id" into fastqc_results """ + ${preCmd} mkdir ${pair_id} fastqc -q ${reads} -o ${pair_id} -t $task.cpus """ } process run_multiqc { - publishDir "${params.out_dir}/multiqc", mode: 'copy' + publishDir "${params.out_dir}/multiqc", mode: "${params.savemode}" tag {"multiqc"} label 'one' @@ -67,6 +68,7 @@ process run_multiqc { file "multiqc_report.html" into multiqc_report """ + ${preCmd} multiqc fastqc_output """ } @@ -98,7 +100,7 @@ process collate_data { */ process run_strip { - publishDir "${params.out_dir}/bbduk", mode: "copy" + publishDir "${params.out_dir}/bbduk", mode: "${params.savemode}" tag { pair_id } input: @@ -125,7 +127,7 @@ process run_strip { * Remove adapter sequences and low quality base pairs with Trimmomatic */ process run_trim { - publishDir "${params.out_dir}/bbduk_trimmed", mode: "copy" + publishDir "${params.out_dir}/bbduk_trimmed", mode: "${params.savemode}" tag { pair_id } input: @@ -138,13 +140,13 @@ process run_trim { """ ${preCmd} trimmomatic PE -threads $task.cpus -trimlog ${pair_id}_concat_stripped_trimmed.log ${pair_id}*_concat_stripped.fq.gz \ - -baseout ${pair_id}_trimmed ILLUMINACLIP:${params.adapter_dir}/${params.adapters}:${params.illuminaClipOptions} \ + -baseout ${pair_id}_trimmed.fq.gz ILLUMINACLIP:${params.adapter_dir}/${params.adapters}:${params.illuminaClipOptions} \ SLIDINGWINDOW:${params.slidingwindow} \ LEADING:${params.leading} TRAILING:${params.trailing} \ MINLEN:${params.minlen} &> ${pair_id}_run.log - mv ${pair_id}_trimmed_1P ${pair_id}_R1_concat_stripped_trimmed.fq.gz - mv ${pair_id}_trimmed_2P ${pair_id}_R2_concat_stripped_trimmed.fq.gz - cat ${pair_id}_trimmed_1U ${pair_id}_trimmed_2U > ${pair_id}_S_concat_stripped_trimmed.fq.gz + mv ${pair_id}_trimmed_1P.fq.gz ${pair_id}_R1_concat_stripped_trimmed.fq.gz + mv ${pair_id}_trimmed_2P.fq.gz ${pair_id}_R2_concat_stripped_trimmed.fq.gz + cat ${pair_id}_trimmed_1U.fq.gz ${pair_id}_trimmed_2U.fq.gz > ${pair_id}_S_concat_stripped_trimmed.fq.gz """ } @@ -153,7 +155,7 @@ process run_trim { * Build assembly with SPAdes */ process run_spadesasm { - publishDir "${params.out_dir}/spades", mode: "copy" + publishDir "${params.out_dir}/spades", mode: "${params.savemode}" tag { pair_id } label 'longtime' @@ -183,7 +185,7 @@ process run_spadesasm { * Map reads to the spades assembly */ process run_bwamem { - publishDir "${params.out_dir}/bwamem", mode: "copy" + publishDir "${params.out_dir}/bwamem", mode: "${params.savemode}" tag { pair_id } label 'longtime' @@ -196,6 +198,7 @@ process run_bwamem { file("${pair_id}_mapped_sorted.bam.bai") into bwamem_results """ + ${preCmd} bwa index ${pair_id}_spades_scaffolds.fasta bwa mem -t $task.cpus ${pair_id}_spades_scaffolds.fasta \ *.fq.gz | samtools sort -o ${pair_id}_mapped_sorted.bam - @@ -208,7 +211,7 @@ process run_bwamem { */ process run_pilon { - publishDir "${params.out_dir}/pilon", mode: "copy" + publishDir "${params.out_dir}/pilon", mode: "${params.savemode}" tag { pair_id } input: @@ -222,6 +225,8 @@ process run_pilon { file "${pair_id}_pilon_spades.fasta" into asms_for_quast """ + ${preCmd} + export _JAVA_OPTIONS=$task.javaopts pilon --threads $task.cpus --genome ${pair_id}_spades_scaffolds.fasta \ --bam ${pair_id}_mapped_sorted.bam --output ${pair_id}_pilon_spades \ --changes --vcfqe &> ${pair_id}_pilon_spades.log @@ -232,7 +237,7 @@ process run_pilon { * Annotation using PROKKA */ process run_prokka { - publishDir "${params.out_dir}/prokka", mode: "copy" + publishDir "${params.out_dir}/prokka", mode: "${params.savemode}" tag { pair_id } input: @@ -257,7 +262,7 @@ process run_prokka { process quast_eval { // The output here is a directory in and of itself // thus not creating a new one - publishDir "${params.out_dir}/", mode: "copy" + publishDir "${params.out_dir}/", mode: "${params.savemode}" tag { pair_id } input: diff --git a/conf/asm_annot_template.config b/conf/asm_annot_template.config index 8f83762..be04dc9 100644 --- a/conf/asm_annot_template.config +++ b/conf/asm_annot_template.config @@ -22,6 +22,7 @@ params.out_dir = "track_three" // General configuration variables params.pwd = "$PWD" params.help = false +params.savemode = "copy" // BBDuk params, has to be absolute paths params.stripgenome = "/work/projects/nn9305k/genome_references/genomes/PhiX/PhiX.fasta" diff --git a/conf/qc_track_template.config b/conf/qc_track_template.config index 9918b16..2a5c547 100644 --- a/conf/qc_track_template.config +++ b/conf/qc_track_template.config @@ -21,6 +21,7 @@ params.out_dir = "track_one" // General configuration variables params.pwd = "$PWD" params.help = false +params.savemode = "link" // Raw fastqc results params.fastqc = "fastqc" diff --git a/conf/slurm.config b/conf/slurm.config index cb82374..0a3d20e 100644 --- a/conf/slurm.config +++ b/conf/slurm.config @@ -27,6 +27,9 @@ process { cpus = 16 withLabel: one {cpus = 1} + //this is mostly for pilon, but can be used elsewhere too + javaopts = '-Xmx12G' + time = { 1.h * task.attempt } withLabel: longtime {time = { 4.h * task.attempt }} } diff --git a/conf/specific_genes_template.config b/conf/specific_genes_template.config index 94a3059..5d29c80 100644 --- a/conf/specific_genes_template.config +++ b/conf/specific_genes_template.config @@ -15,6 +15,7 @@ params.out_dir = "track_two" // General configuration variables params.pwd = "$PWD" params.help = false +params.savemode = "copy" params.threads = 1 diff --git a/conf/standard.config b/conf/standard.config index 5042526..7944644 100644 --- a/conf/standard.config +++ b/conf/standard.config @@ -8,4 +8,7 @@ process { executor = "local" cpus = 1 maxForks = 2 + //this is mostly for pilon, but can be used elsewhere too + javaopts = '' + } diff --git a/qc_track.nf b/qc_track.nf index 59c9f03..1a09a16 100644 --- a/qc_track.nf +++ b/qc_track.nf @@ -37,7 +37,7 @@ Channel // Second is to send all of through fastqc process run_fastqc { - publishDir "${params.out_dir}/${params.fastqc}", mode: 'copy' + publishDir "${params.out_dir}/${params.fastqc}", mode: "${params.savemode}" tag {pair_id} label 'one' @@ -48,13 +48,14 @@ process run_fastqc { file "$pair_id" into fastqc_results """ + ${preCmd} mkdir ${pair_id} fastqc -q ${reads} -o ${pair_id} -t $task.cpus """ } process run_multiqc { - publishDir "${params.out_dir}/multiqc", mode: 'copy' + publishDir "${params.out_dir}/multiqc", mode: "${params.savemode}" tag {"multiqc"} label 'one' @@ -65,6 +66,7 @@ process run_multiqc { file "multiqc_report.html" into multiqc_report """ + ${preCmd} multiqc fastqc_output """ } diff --git a/specific_genes.nf b/specific_genes.nf index 4535760..b3c84ff 100644 --- a/specific_genes.nf +++ b/specific_genes.nf @@ -66,7 +66,7 @@ process collate_data { // Download database database process run_ariba_mlst_prep { - publishDir params.out_dir + "/" + params.mlst_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.mlst_results}", mode: "${params.savemode}" tag {'Dowloading schema'} label 'one' @@ -85,7 +85,7 @@ process run_ariba_mlst_prep { // Run ariba on each dataset process run_ariba_mlst_pred { - publishDir params.out_dir + "/" + params.mlst_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.mlst_results}", mode: "${params.savemode}" tag {pair_id} input: @@ -109,7 +109,7 @@ process run_ariba_mlst_pred { // Summarize MLST results process run_ariba_mlst_summarize { - publishDir params.out_dir + "/" + params.mlst_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.mlst_results}", mode: "${params.savemode}" tag {'Summarizing mlst'} label 'one' @@ -134,7 +134,7 @@ process run_ariba_mlst_summarize { // These three processes are for AMR prediction process run_ariba_amr_prep { - publishDir params.out_dir + "/" + params.amr_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.amr_results}", mode: "${params.savemode}" tag {'Dowloading AMR data'} label 'one' @@ -152,7 +152,7 @@ process run_ariba_amr_prep { } process run_ariba_amr_pred { - publishDir params.out_dir + "/" + params.amr_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.amr_results}", mode: "${params.savemode}" tag{pair_id} input: @@ -177,7 +177,7 @@ process run_ariba_amr_pred { // Summarize AMR results process run_ariba_amr_summarize { - publishDir params.out_dir + "/" + params.amr_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.amr_results}", mode: "${params.savemode}" tag{'Summarizing AMR'} label 'one' @@ -198,7 +198,7 @@ process run_ariba_amr_summarize { // These three processes are for virulence prediction process run_ariba_vir_prep { - publishDir params.out_dir + "/" + params.vir_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.vir_results}", mode: "${params.savemode}" tag{'Downloading virulence data'} label 'one' @@ -216,7 +216,7 @@ process run_ariba_vir_prep { } process run_ariba_vir_pred { - publishDir params.out_dir + "/" + params.vir_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.vir_results}", mode: "${params.savemode}" tag{pair_id} input: @@ -241,7 +241,7 @@ process run_ariba_vir_pred { // Summarize virulence results process run_ariba_vir_summarize { - publishDir params.out_dir + "/" + params.vir_results, mode: 'copy' + publishDir "${params.out_dir}" + "/" + "${params.vir_results}", mode: "${params.savemode}" tag{'Summarizing virulence'} label 'one'