Merge pull request #346 from sanger-tol/dp24_new_dev

Dp24 new dev

.github/workflows/ci.yml

@@ -22,48 +22,21 @@ jobs:
     name: Run pipeline with test data
     # Only run on push if this is the nf-core dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'sanger-tol/treeval') }}"
-    runs-on: [ubuntu-latest] # Let's see if Pretext errors are a architecture thing
+    runs-on: [ubuntu-latest]
     strategy:
       matrix:
         NXF_VER:
           - "22.10.1"
           - "latest-everything"
     steps:
-      - name: Get branch names
-        # Pulls the names of current branches in repo
-        # steps.branch-names.outputs.current_branch is used later and returns the name of the branch the PR is made FROM not to
-        id: branch-names
-        uses: tj-actions/branch-names@v8
+      - name: Check out pipeline code
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Setup apptainer
-        uses: eWaterCycle/setup-apptainer@main
-
-      - name: Set up Singularity
-        run: |
-          mkdir -p $NXF_SINGULARITY_CACHEDIR
-          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
-
-      - name: Install Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install nf-core
-        run: |
-          pip install nf-core
-
-      - name: NF-Core Download - download singularity containers
-        # Forcibly download repo on active branch and download SINGULARITY containers into the CACHE dir if not found
-        # Must occur after singularity install or will crash trying to dl containers
-        # Zip up this fresh download and run the checked out version
-        run: |
-          nf-core download sanger-tol/treeval --revision ${{ steps.branch-names.outputs.current_branch }} --compress none -d --force --outdir sanger-treeval --container-cache-utilisation amend --container-system singularity
-
       - name: Download Tiny test data
         # Download A fungal test data set that is full enough to show some real output.
         run: |
@@ -72,4 +45,4 @@ jobs:
       - name: Singularity - Run FULL pipeline with test data
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ./sanger-treeval/${{ steps.branch-names.outputs.current_branch }}/main.nf -profile test_github,singularity --outdir ./Sing-Full
+          nextflow run ${GITHUB_WORKSPACE} -profile test_github,docker --outdir ./Sing-Full

.gitignore

@@ -12,3 +12,8 @@ output/
 null/
 error
 out
+OscSUBSET-TEST
+OscSUBSET-TEST-MODULE-UPDATE
+OscSUBSET-TEST-MODULE-UPDATE-3
+TreeValTinyData
+yaml

CHANGELOG.md

@@ -118,13 +118,6 @@ This builds on the initial release by adding subworkflows which generate kmer ba
 - Fix a bug in build_alignment_blocks.py to avoid indexing errors happening in large genomes.
 - Change output BEDGRAPH from EXTRACT_TELO module.
 
-#### Hot Fix 1
-
-- Generate CRAM CSV fix to allow for multi-readgroup cram files
-- Removing KMER_READCOV
-- tmp directory was being used
-- Output file adjustment (names and location)
-
 ### Parameters
 
 | Old Parameter | New Parameter |

CITATIONS.md

@@ -34,10 +34,6 @@
 
   > Durand, N.C. et al. 2016. ‘Juicer provides a one-click system for analyzing loop-resolution hi-C experiments’, Cell Systems, 3(1), pp. 95–98. doi:10.1016/j.cels.2016.07.002.
 
-- [Merqury_FK](https://github.com/thegenemyers/MERQURY.FK)
-
-  > Myers, G., Rhie, A. (2024). MerquryFK & KatFK. [online]. https://github.com/thegenemyers/MERQURY.FK. (Accessed on 20 September 2024).
-
 - [Minimap2](https://pubmed.ncbi.nlm.nih.gov/34623391/)
 
   > Li, H. 2021. ‘New strategies to improve MINIMAP2 alignment accuracy’, Bioinformatics, 37(23), pp. 4572–4574. doi:10.1093/bioinformatics/btab705.
@@ -76,7 +72,7 @@
 
 - [Samtools](https://pubmed.ncbi.nlm.nih.gov/33590861/)
 
-  > Danecek P, Bonfield JK, Liddle J, Marshall J, Ohan V, Pollard MO, Whitwham A, Keane T, McCarthy SA, Davies RM, Li H. Twelve years of SAMtools and BCFtools. Gigascience. 2021 Feb 16;10(2):giab008. doi: 10.1093/gigascience/giab008. PMID: 33590861; PMCID: PMC7931819.
+  > Di Tommaso, Paolo, et al. 2017. “Nextflow Enables Reproducible Computational Workflows.” Nature Biotechnology, 35(4), pp. 316–19, https://doi.org/10.1038/nbt.3820.
 
 - [SeqTK](https://github.com/lh3/seqtk)
 

main.nf

@@ -25,7 +25,6 @@ WorkflowMain.initialise( workflow, params, log )
 include { TREEVAL           } from './workflows/treeval'
 include { TREEVAL_RAPID     } from './workflows/treeval_rapid'
 include { TREEVAL_RAPID_TOL } from './workflows/treeval_rapid_tol'
-include { TREEVAL_JBROWSE   } from './workflows/treeval_jbrowse'
 
 //
 // WORKFLOW: RUN MAIN PIPELINE GENERATING ALL OUTPUT
@@ -48,15 +47,6 @@ workflow SANGERTOL_TREEVAL_RAPID_TOL {
         TREEVAL_RAPID_TOL ()
 }
 
-//
-// WORKFLOW: RUN ONLY THE SUBWORKFLOWS REQUIRED FOR JBROWSE UPLOAD
-//              - THIS IS TO COMPLEMENT A NEW PROCESS WHERE MAJORITY OF TICKETS WILL BE RC
-//                  AND GET REQUESTED FOR FULL
-//
-workflow SANGERTOL_TREEVAL_JBROWSE {
-    TREEVAL_JBROWSE ()
-}
-
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     RUN ALL WORKFLOWS
@@ -78,10 +68,6 @@ workflow RAPID_TOL {
         SANGERTOL_TREEVAL_RAPID_TOL ()
 }
 
-workflow JBROWSE {
-    SANGERTOL_TREEVAL_JBROWSE ()
-}
-
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     THE END

modules/local/avgcov.nf

@@ -4,8 +4,8 @@ process AVGCOV {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
     tuple val(meta), path(bedfile)

modules/local/bamtobed_sort.nf

@@ -6,11 +6,6 @@ process BAMTOBED_SORT {
         'https://depot.galaxyproject.org/singularity/mulled-v2-9d3a458f6420e5712103ae2af82c94d26d63f059:60b54b43045e8cf39ba307fd683c69d4c57240ce-0' :
         'biocontainers/mulled-v2-9d3a458f6420e5712103ae2af82c94d26d63f059:60b54b43045e8cf39ba307fd683c69d4c57240ce-0' }"
 
-    // Exit if running this module with -profile conda / -profile mamba
-    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
-        error "BAMTOBED_SORT module does not support Conda. Please use Docker / Singularity instead."
-    }
-
     input:
     tuple val(meta), path(bam)
 

modules/local/concatblocks.nf

@@ -4,8 +4,8 @@ process CONCATBLOCKS {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+            'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+            'docker.io/ubuntu:20.04' }"
 
     input:
     tuple val(meta), path(mergeblocks)

modules/local/cram_filter_align_bwamem2_fixmate_sort.nf

@@ -6,11 +6,6 @@ process CRAM_FILTER_ALIGN_BWAMEM2_FIXMATE_SORT {
         'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
         'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
 
-    // Exit if running this module with -profile conda / -profile mamba
-    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
-        error "CRAM_FILTER_ALIGN_BWAMEM2_FIXMATE_SORT module does not support Conda. Please use Docker / Singularity instead."
-    }
-
     input:
     tuple val(meta), path(cramfile), path(cramindex), val(from), val(to), val(base), val(chunkid), val(rglines), val(bwaprefix), path(reference)
 

modules/local/cram_filter_minimap2_filter5end_fixmate_sort.nf

@@ -6,11 +6,6 @@ process CRAM_FILTER_MINIMAP2_FILTER5END_FIXMATE_SORT {
         'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
         'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
 
-    // Exit if running this module with -profile conda / -profile mamba
-    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
-        error "CRAM_FILTER_MINIMAP2_FILTER5END_FIXMATE_SORT module does not support Conda. Please use Docker / Singularity instead."
-    }
-
     input:
     tuple val(meta), path(cramfile), path(cramindex), val(from), val(to), val(base), val(chunkid), val(rglines), val(ref), path(reference)
 

modules/local/extract_buscogene.nf

@@ -4,8 +4,8 @@ process EXTRACT_BUSCOGENE {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
 
     input:

modules/local/extract_cov_iden.nf

@@ -4,15 +4,15 @@ process EXTRACT_COV_IDEN {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
-    tuple val(meta), file("*.bed")  , emit: punchlist
-    path "versions.yml"             , emit: versions
+    tuple val( meta ), file( "*.bed" )  , emit: punchlist
+    path "versions.yml"                 , emit: versions
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}_${meta.type}_punchlist"

modules/local/extract_repeat.nf

@@ -8,11 +8,11 @@ process EXTRACT_REPEAT {
         'biocontainers/perl:5.26.2' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
-    tuple val(meta), path("*.bed")  , emit: bed
-    path "versions.yml"             , emit: versions
+    tuple val( meta ), path( "*.bed" )  , emit: bed
+    path "versions.yml"                 , emit: versions
 
     when:
     task.ext.when == null || task.ext.when

modules/local/extract_telo.nf

@@ -4,16 +4,16 @@ process EXTRACT_TELO {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
-    tuple val(meta), file("*bed")     , emit: bed
-    tuple val(meta), file("*bedgraph"), emit: bedgraph
-    path "versions.yml"               , emit: versions
+    tuple val( meta ), file( "*bed" )   , emit: bed
+    tuple val( meta ), file("*bedgraph"), emit: bedgraph
+    path "versions.yml"                 , emit: versions
 
     shell:
     def prefix  = task.ext.prefix ?: "${meta.id}"

modules/local/find_telomere_regions.nf

@@ -4,18 +4,13 @@ process FIND_TELOMERE_REGIONS {
 
     container 'quay.io/sanger-tol/telomere:0.0.1-c1'
 
-    // Exit if running this module with -profile conda / -profile mamba
-    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
-        error "FIND_TELOMERE_REGIONS module does not support Conda. Please use Docker / Singularity instead."
-    }
-
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
     val (telomereseq)
 
     output:
-    tuple val(meta), file("*.telomere") , emit: telomere
-    path "versions.yml"                 , emit: versions
+    tuple val( meta ), file( "*.telomere" ) , emit: telomere
+    path "versions.yml"                     , emit: versions
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/find_telomere_windows.nf

@@ -3,16 +3,17 @@ process FIND_TELOMERE_WINDOWS {
     label 'process_low'
 
     conda "bioconda::java-jdk=8.0.112"
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+    container "${ workflow.containerEngine == 'singularity' &&
+                    !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/java-jdk:8.0.112--1' :
         'biocontainers/java-jdk:8.0.112--1' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
-    tuple val(meta), file("*.windows") , emit: windows
-    path "versions.yml"                , emit: versions
+    tuple val( meta ), file( "*.windows" ) , emit: windows
+    path "versions.yml"                    , emit: versions
 
     script:
     def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/gap_length.nf

@@ -4,15 +4,15 @@ process GAP_LENGTH {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
-    tuple val(meta), file("*bedgraph")  , emit: bedgraph
-    path "versions.yml"                 , emit: versions
+    tuple val( meta ), file( "*bedgraph" )  , emit: bedgraph
+    path "versions.yml"                     , emit: versions
 
     shell:
     def prefix = task.ext.prefix ?: "${meta.id}"

modules/local/generate_cram_csv.nf

@@ -6,11 +6,6 @@ process GENERATE_CRAM_CSV {
         'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
         'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
 
-    // Exit if running this module with -profile conda / -profile mamba
-    if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {
-        error "GENERATE_CRAM_CSV module does not support Conda. Please use Docker / Singularity instead."
-    }
-
     input:
     tuple val(meta), path(crampath)
 

modules/local/get_largest_scaff.nf

@@ -5,11 +5,11 @@ process GET_LARGEST_SCAFF {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
     env largest_scaff   , emit: scaff_size

modules/local/get_paired_contact_bed.nf

@@ -4,15 +4,15 @@ process GET_PAIRED_CONTACT_BED {
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
-    tuple val(meta), path(file)
+    tuple val( meta ), path( file )
 
     output:
-    tuple val(meta), file("*bed")   , emit: bed
-    path "versions.yml"             , emit: versions
+    tuple val( meta ), file( "*bed" )   , emit: bed
+    path "versions.yml"                 , emit: versions
 
     script:
     def pulled = '-T sort_tmp'

modules/local/getminmaxpunches.nf

@@ -4,16 +4,16 @@ process GETMINMAXPUNCHES{
 
     conda "conda-forge::coreutils=9.1"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
-        'docker.io/ubuntu:20.04' }"
+    'https://depot.galaxyproject.org/singularity/ubuntu:20.04' :
+    'docker.io/ubuntu:20.04' }"
 
     input:
     tuple val(meta), path(bedfile)
 
     output:
-    tuple val(meta), path ('*zero.bed')   , optional: true    , emit: min
-    tuple val(meta), path ('*max.bed')    , optional: true    , emit: max
-    path "versions.yml"                   , emit: versions
+    tuple val(meta), path ( '*zero.bed' )   , optional: true    , emit: min
+    tuple val(meta), path ( '*max.bed' )    , optional: true    , emit: max
+    path "versions.yml"                     , emit: versions
 
     shell:
     def VERSION = "9.1" // WARN: Version information not provided by tool on CLI. Please update this string when bumping container versions.