Merge pull request #319 from sanger-tol/dp24_steps

Dp24 steps

.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
     name: Run pipeline with test data
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/treeval') }}"
-    runs-on: ubuntu2204-8c
+    runs-on: [ubuntu-latest] # Let's see if Pretext errors are a architecture thing
     strategy:
       matrix:
         NXF_VER:

CHANGELOG.md

@@ -40,11 +40,20 @@ This builds on the initial release by adding subworkflows which generate kmer ba
 - Fix a bug in build_alignment_blocks.py to avoid indexing errors happening in large genomes.
 - Change output BEDGRAPH from EXTRACT_TELO module.
 
+#### Hot Fix 1
+
+- Adding support for multi-library cram input.
+
+#### Hot Fix 2
+
+- Adding support to select subworkflows to use in pipeline run.
+
 ### Parameters
 
 | Old Parameter | New Parameter |
 | ------------- | ------------- |
 | -             | --juicer      |
+| -             | --steps       |
 
 ### Software dependencies
 

CITATIONS.md

@@ -34,6 +34,10 @@
 
   > Durand, N.C. et al. 2016. ‘Juicer provides a one-click system for analyzing loop-resolution hi-C experiments’, Cell Systems, 3(1), pp. 95–98. doi:10.1016/j.cels.2016.07.002.
 
+- [Merqury_FK](https://github.com/thegenemyers/MERQURY.FK)
+
+  > Myers, G., Rhie, A. (2024). MerquryFK & KatFK. [online]. https://github.com/thegenemyers/MERQURY.FK. (Accessed on 20 September 2024).
+
 - [Minimap2](https://pubmed.ncbi.nlm.nih.gov/34623391/)
 
   > Li, H. 2021. ‘New strategies to improve MINIMAP2 alignment accuracy’, Bioinformatics, 37(23), pp. 4572–4574. doi:10.1093/bioinformatics/btab705.
@@ -72,7 +76,7 @@
 
 - [Samtools](https://pubmed.ncbi.nlm.nih.gov/33590861/)
 
-  > Di Tommaso, Paolo, et al. 2017. “Nextflow Enables Reproducible Computational Workflows.” Nature Biotechnology, 35(4), pp. 316–19, https://doi.org/10.1038/nbt.3820.
+  > Danecek P, Bonfield JK, Liddle J, Marshall J, Ohan V, Pollard MO, Whitwham A, Keane T, McCarthy SA, Davies RM, Li H. Twelve years of SAMtools and BCFtools. Gigascience. 2021 Feb 16;10(2):giab008. doi: 10.1093/gigascience/giab008. PMID: 33590861; PMCID: PMC7931819.
 
 - [SeqTK](https://github.com/lh3/seqtk)
 

main.nf

@@ -25,6 +25,7 @@ WorkflowMain.initialise( workflow, params, log )
 include { TREEVAL           } from './workflows/treeval'
 include { TREEVAL_RAPID     } from './workflows/treeval_rapid'
 include { TREEVAL_RAPID_TOL } from './workflows/treeval_rapid_tol'
+include { TREEVAL_JBROWSE   } from './workflows/treeval_jbrowse'
 
 //
 // WORKFLOW: RUN MAIN PIPELINE GENERATING ALL OUTPUT
@@ -47,6 +48,15 @@ workflow SANGERTOL_TREEVAL_RAPID_TOL {
         TREEVAL_RAPID_TOL ()
 }
 
+//
+// WORKFLOW: RUN ONLY THE SUBWORKFLOWS REQUIRED FOR JBROWSE UPLOAD
+//              - THIS IS TO COMPLEMENT A NEW PROCESS WHERE MAJORITY OF TICKETS WILL BE RC
+//                  AND GET REQUESTED FOR FULL
+//
+workflow SANGERTOL_TREEVAL_JBROWSE {
+    TREEVAL_JBROWSE ()
+}
+
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     RUN ALL WORKFLOWS
@@ -68,6 +78,10 @@ workflow RAPID_TOL {
         SANGERTOL_TREEVAL_RAPID_TOL ()
 }
 
+workflow JBROWSE {
+    SANGERTOL_TREEVAL_JBROWSE ()
+}
+
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     THE END

modules/local/avgcov.nf

@@ -4,8 +4,8 @@ process AVGCOV {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
     tuple val(meta), path(bedfile)

modules/local/bamtobed_sort.nf

@@ -6,6 +6,11 @@ process BAMTOBED_SORT {
         'https://depot.galaxyproject.org/singularity/mulled-v2-9d3a458f6420e5712103ae2af82c94d26d63f059:60b54b43045e8cf39ba307fd683c69d4c57240ce-0' :
         'biocontainers/mulled-v2-9d3a458f6420e5712103ae2af82c94d26d63f059:60b54b43045e8cf39ba307fd683c69d4c57240ce-0' }"
 
+    // Exit if running this module with -profile conda / -profile mamba
+    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
+        error "BAMTOBED_SORT module does not support Conda. Please use Docker / Singularity instead."
+    }
+
     input:
     tuple val(meta), path(bam)
 

modules/local/chunkfasta.nf

@@ -4,8 +4,8 @@ process CHUNKFASTA {
 
     conda "conda-forge::pyfasta=0.5.2-1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/pyfasta:0.5.2--py_1' :
-    'biocontainers/pyfasta:0.5.2--py_1' }"
+        'https://depot.galaxyproject.org/singularity/pyfasta:0.5.2--py_1' :
+        'biocontainers/pyfasta:0.5.2--py_1' }"
 
     input:
     tuple val(meta), path('input.fasta')

modules/local/concatblocks.nf

@@ -4,8 +4,8 @@ process CONCATBLOCKS {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-            'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-            'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
     tuple val(meta), path(mergeblocks)

modules/local/cram_filter_align_bwamem2_fixmate_sort.nf

@@ -6,6 +6,11 @@ process CRAM_FILTER_ALIGN_BWAMEM2_FIXMATE_SORT {
         'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
         'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
 
+    // Exit if running this module with -profile conda / -profile mamba
+    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
+        error "CRAM_FILTER_ALIGN_BWAMEM2_FIXMATE_SORT module does not support Conda. Please use Docker / Singularity instead."
+    }
+
     input:
     tuple val(meta), path(cramfile), path(cramindex), val(from), val(to), val(base), val(chunkid), val(rglines), val(bwaprefix), path(reference)
 

modules/local/cram_filter_minimap2_filter5end_fixmate_sort.nf

@@ -6,6 +6,11 @@ process CRAM_FILTER_MINIMAP2_FILTER5END_FIXMATE_SORT {
         'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
         'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
 
+    // Exit if running this module with -profile conda / -profile mamba
+    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
+        error "CRAM_FILTER_MINIMAP2_FILTER5END_FIXMATE_SORT module does not support Conda. Please use Docker / Singularity instead."
+    }
+
     input:
     tuple val(meta), path(cramfile), path(cramindex), val(from), val(to), val(base), val(chunkid), val(rglines), val(ref), path(reference)
 

modules/local/extract_buscogene.nf

@@ -4,8 +4,8 @@ process EXTRACT_BUSCOGENE {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
 
     input:

modules/local/extract_cov_iden.nf

@@ -4,15 +4,15 @@ process EXTRACT_COV_IDEN {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
-    tuple val( meta ), file( "*.bed" )  , emit: punchlist
-    path "versions.yml"                 , emit: versions
+    tuple val(meta), file("*.bed")  , emit: punchlist
+    path "versions.yml"             , emit: versions
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}_${meta.type}_punchlist"

modules/local/extract_repeat.nf

@@ -8,11 +8,11 @@ process EXTRACT_REPEAT {
         'biocontainers/perl:5.26.2' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
-    tuple val( meta ), path( "*.bed" )  , emit: bed
-    path "versions.yml"                 , emit: versions
+    tuple val(meta), path("*.bed")  , emit: bed
+    path "versions.yml"             , emit: versions
 
     when:
     task.ext.when == null || task.ext.when

modules/local/extract_telo.nf

@@ -4,16 +4,16 @@ process EXTRACT_TELO {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
-    tuple val( meta ), file( "*bed" )   , emit: bed
-    tuple val( meta ), file("*bedgraph"), emit: bedgraph
-    path "versions.yml"                 , emit: versions
+    tuple val(meta), file("*bed")     , emit: bed
+    tuple val(meta), file("*bedgraph"), emit: bedgraph
+    path "versions.yml"               , emit: versions
 
     shell:
     def prefix  = task.ext.prefix ?: "${meta.id}"

modules/local/find_telomere_regions.nf

@@ -4,13 +4,18 @@ process FIND_TELOMERE_REGIONS {
 
     container 'docker.io/library/gcc:10.4.0'
 
+    // Exit if running this module with -profile conda / -profile mamba
+    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
+        error "FIND_TELOMERE_REGIONS module does not support Conda. Please use Docker / Singularity instead."
+    }
+
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
     val (telomereseq)
 
     output:
-    tuple val( meta ), file( "*.telomere" ) , emit: telomere
-    path "versions.yml"                     , emit: versions
+    tuple val(meta), file("*.telomere") , emit: telomere
+    path "versions.yml"                 , emit: versions
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/find_telomere_windows.nf

@@ -3,17 +3,16 @@ process FIND_TELOMERE_WINDOWS {
     label 'process_low'
 
     conda "bioconda::java-jdk=8.0.112"
-    container "${ workflow.containerEngine == 'singularity' &&
-                    !task.ext.singularity_pull_docker_container ?
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/java-jdk:8.0.112--1' :
         'biocontainers/java-jdk:8.0.112--1' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
-    tuple val( meta ), file( "*.windows" ) , emit: windows
-    path "versions.yml"                    , emit: versions
+    tuple val(meta), file("*.windows") , emit: windows
+    path "versions.yml"                , emit: versions
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/fkutils/fkprof/main.nf

@@ -6,13 +6,18 @@ process FKUTILS_FKPROF {
     'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
     'docker.io/ubuntu:20.04' }"
 
+    // Exit if running this module with -profile conda / -profile mamba
+    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
+        error "FKUTILS_FKPROF module does not support Conda. Please use Docker / Singularity instead."
+    }
+
     input:
-    tuple val( meta ), path( reference )
-    tuple val( meta2 ), path( ktab )
+    tuple val(meta), path(reference)
+    tuple val(meta2), path(ktab)
 
     output:
-    tuple val( meta ), file( "*bed" ),  emit: bed
-    path "versions.yml",                emit: versions
+    tuple val(meta), file("*bed"),  emit: bed
+    path "versions.yml",            emit: versions
 
     script:
     def args    = task.ext.args     ?: ""

modules/local/gap_length.nf

@@ -4,15 +4,15 @@ process GAP_LENGTH {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
-    tuple val( meta ), file( "*bedgraph" )  , emit: bedgraph
-    path "versions.yml"                     , emit: versions
+    tuple val(meta), file("*bedgraph")  , emit: bedgraph
+    path "versions.yml"                 , emit: versions
 
     shell:
     def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/generate_cram_csv.nf

@@ -2,7 +2,14 @@ process GENERATE_CRAM_CSV {
     tag "${meta.id}"
     label 'process_tiny'
 
-    container 'quay.io/sanger-tol/cramfilter_bwamem2_minimap2_samtools_perl:0.001-c1'
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
+        'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
+
+    // Exit if running this module with -profile conda / -profile mamba
+    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
+        error "GENERATE_CRAM_CSV module does not support Conda. Please use Docker / Singularity instead."
+    }
 
     input:
     tuple val(meta), path(crampath)

modules/local/get_largest_scaff.nf

@@ -5,11 +5,11 @@ process GET_LARGEST_SCAFF {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
     env largest_scaff   , emit: scaff_size

modules/local/get_paired_contact_bed.nf

@@ -4,15 +4,15 @@ process GET_PAIRED_CONTACT_BED {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val( meta ), path( file )
+    tuple val(meta), path(file)
 
     output:
-    tuple val( meta ), file( "*bed" )   , emit: bed
-    path "versions.yml"                 , emit: versions
+    tuple val(meta), file("*bed")   , emit: bed
+    path "versions.yml"             , emit: versions
 
     script:
     def pulled = '-T sort_tmp'

modules/local/getminmaxpunches.nf

@@ -4,16 +4,16 @@ process GETMINMAXPUNCHES{
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-    'docker.io/ubuntu:20.04' }"
+        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+        'docker.io/ubuntu:20.04' }"
 
     input:
     tuple val(meta), path(bedfile)
 
     output:
-    tuple val(meta), path ( '*zero.bed' )   , optional: true    , emit: min
-    tuple val(meta), path ( '*max.bed' )    , optional: true    , emit: max
-    path "versions.yml"                     , emit: versions
+    tuple val(meta), path ('*zero.bed')   , optional: true    , emit: min
+    tuple val(meta), path ('*max.bed')    , optional: true    , emit: max
+    path "versions.yml"                   , emit: versions
 
     shell:
     def VERSION = "9.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions.