Dev to main

.github/workflows/ci.yml

@@ -29,50 +29,20 @@ jobs:
           - "22.10.1"
           - "latest-everything"
     steps:
-      - name: Get branch names
-        # Pulls the names of current branches in repo
-        # steps.branch-names.outputs.current_branch is used later and returns the name of the branch the PR is made FROM not to
-        id: branch-names
-        uses: tj-actions/branch-names@v8
-
       - name: Check out pipeline code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 
-      - name: Setup apptainer
-        uses: eWaterCycle/setup-apptainer@main
-
-      - name: Set up Singularity
-        run: |
-          mkdir -p $NXF_SINGULARITY_CACHEDIR
-          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
-
-      - name: Install Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install nf-core
-        run: |
-          pip install nf-core
-
-      - name: NF-Core Download - download singularity containers
-        # Forcibly download repo on active branch and download SINGULARITY containers into the CACHE dir if not found
-        # Must occur after singularity install or will crash trying to dl containers
-        # Zip up this fresh download and run the checked out version
-        run: |
-          nf-core download sanger-tol/curationpretext --revision ${{ steps.branch-names.outputs.current_branch }} --compress none -d --force --outdir sanger-curationpretext --container-cache-utilisation amend --container-system singularity
-
       - name: Download test data
         # Download A fungal test data set that is full enough to show some real output.
         run: |
           curl https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData.tar.gz | tar xzf -
 
-      - name: Singularity - Run ALL_FILES pipeline with test data
+      - name: Run ALL_FILES pipeline with test data
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ./sanger-curationpretext/${{ steps.branch-names.outputs.current_branch }}/main.nf -profile test,singularity --outdir ./Sing-res
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results

CHANGELOG.md

@@ -3,6 +3,88 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [[1.1.0](https://github.com/sanger-tol/curationpretext/releases/tag/1.1.0)] - UNSC Delphi - [2024-12-09]
+
+### Added
+
+- Added map_order so that the output maps are defaulted to unsorted and can be selected as sorted.
+- Updating all modules.
+- Removing Anaconda 'defaults' channel.
+- Updating local module containers.
+- Update to LICENSE and CITATIONS files.
+- Update algorithms at play for memory allocation, particulary minimap2.
+- Parity update to TreeVal as the mapping subworkflow is based on the treeval implementation.
+- Fixed some version output being generated incorrectly.
+
+### Paramters
+
+| Old Version | New Versions |
+| - | --map_order |
+
+### Software Dependencies
+
+Note, since the pipeline is using Nextflow DSL2, each process will be run with its own Biocontainer. This means that on occasion it is entirely possible for the pipeline to be using different versions of the same tool. However, the overall software dependency changes compared to the last release have been listed below for reference.
+
+| Module                                       | Old Version   | New Versions               |
+| -------------------------------------------- | ------------- | -------------------------- |
+| get_avcov                                    | -             | 1.0.0                      |
+| bamtobed_sort ( bedtools + samtools )        | 2.31.0 + 1.17 | 2.31.1 + 1.17              |
+| bedtools ( all modules)                      | 2.31.1        | -                          |
+| bwamem2 index                                | -             | 2.2.1                      |
+| cram_filter_align_bwamem2_fixmate_sort       | -             |                            |
+| ^ ( samtools + bwamem2 ) ^                   | 1.17 + 2.2.1  | -                          |
+| cram_filter_minimap2_filter5end_fixmate_sort | -             |                            |
+| ^ ( samtools + minimap2 ) ^                  | 1.17 + 2.24   | -                          |
+| custom_dumpsoftwareversions                  | -             | Python 3.11.7 + yaml 5.4.1 |
+| extract_cov_id ( coreutils )                 | 9.1           | 9.3                        |
+| extract_repeat ( perl )                      | 5.26.2        | -                          |
+| extract_telo ( coreutils )                   | -             | 9.1                        |
+| find_telomere_regions ( gcc )                | 7.1.0         | 7.1.0 + 1.0                |
+| find_telomere_windows ( java-jdk )           | 8.0.112       | 8.0.112 + 1.0              |
+| findhalfcoverage ( python )                  | -             | Python 3.9.1 + 1.0         |
+| gap_length ( coreutils )                     | 9.1           | -                          |
+| generate_cram_csv ( samtools )               | 1.17          | -                          |
+| get_largest_scaff ( coreutils )              | 9.1           | -                          |
+| getminmaxpunches ( coreutils )               | 9.1           | -                          |
+| graphoverallcoverage ( perl )                | -             | 5.26.2 + 1.0               |
+| gnu-sort                                     | 8.25          | 9.3                        |
+| longreadcoveragescalelog                     | -             | Python 3.9.1 + 1.0         |
+| minimap2 + samtools (align, map)             |               | 2.28-r1209 + 1.20          |
+| pretextmap + samtools                        | 0.1.9 + 1.18  | 0.1.9\* + 1.20             |
+| pretextgraph                                 | 0.0.4         | 0.0.6                      |
+| pretextsnapshot + UCSC                       | 0.0.6b + 447  | 0.0.4 (official version)   |
+| rename_ids ( coreutils )                     | -             | 9.1                        |
+| replace_dots ( coreutils )                   | -             | 9.1                        |
+| seqtk                                        | 1.4           | 1.4-r122                   |
+| samtools (faidx,merge,sort,view)             | 1.18          | 1.21                       |
+| ucsc                                         | 445           | 469                        |
+| windowmasker (blast)                         | -             | 2.14.0 + 1.0.0             |
+
+Even modules which have not had a version bump have indeed been updated through NF-core to remove defaults.
+
+Some modules now have two versions, the new addition is the script version rather than just the dependency version.
+
+## [[1.0.1](https://github.com/sanger-tol/curationpretext/releases/tag/1.0.1)] - UNSC Cradle H1 - [2024-10-24]
+
+## Added
+
+- Ability for end users to select "sorted" or "unsorted" (default) for the pretext maps.
+- Adds a container for find_telomere.
+
+### Paramters
+
+| Old Version | New Versions |
+| ----------- | ------------ |
+|             | --map_order  |
+
+### Software Dependencies
+
+No updates to dependency versions
+
+### Dependencies
+
+### Deprecated
+
 ## [[1.0.0](https://github.com/sanger-tol/curationpretext/releases/tag/1.0.0)] - UNSC Cradle - [2024-02-22]
 
 ### Added

CITATION.cff

@@ -2,7 +2,7 @@
 # Visit https://bit.ly/cffinit to generate yours today!
 
 cff-version: 1.2.0
-title: sanger-tol/curationpretext v1.0.0
+title: "sanger-tol/curationpretext"
 message: >-
     If you use this software, please cite it using the
     metadata from this file.
@@ -12,23 +12,11 @@ authors:
       family-names: Pointon
       affiliation: Wellcome Sanger Institute
       orcid: "https://orcid.org/0000-0003-2949-6719"
-    - given-names: Matthieu
-      family-names: Muffato
-      affiliation: Wellcome Sanger Institute
-      orcid: "https://orcid.org/0000-0002-7860-3560"
-    - given-names: Ying
-      family-names: Sims
-      affiliation: Wellcome Sanger Institute
-      orcid: "https://orcid.org/0000-0003-4765-4872"
-    - given-names: William
-      family-names: Eagles
-      affiliation: Wellcome Sanger Institute
-      orcid: "https://orcid.org/0009-0006-9956-0404"
 identifiers:
     - type: doi
-      value: 10.5281/zenodo.XXXXXXX
+      value: 10.5281/zenodo.12773958
 repository-code: "https://github.com/sanger-tol/curationpretext"
 license: MIT
-commit: TODO
-version: 1.0.0
-date-released: "2024-07-18"
+version: 1.1.0
+date-released: "2024-12-13"
+url: "https://pipelines.tol.sanger.ac.uk/curationpretext"

CITATIONS.md

@@ -72,9 +72,9 @@
 
 ## Software packaging/containerisation tools
 
-- [Anaconda](https://anaconda.com)
+- [Conda](https://conda.org/)
 
-  > Anaconda Software Distribution. Computer software. Vers. 2-2.4.0. Anaconda, Nov. 2016. Web.
+  > conda contributors. conda: A system-level, binary package and environment manager running on all major operating systems and platforms. Computer software. https://github.com/conda/conda
 
 - [Bioconda](https://pubmed.ncbi.nlm.nih.gov/29967506/)
 

README.md

@@ -1,5 +1,8 @@
 # ![sanger-tol/curationpretext](docs/images/curationpretext-light.png#gh-light-mode-only) ![sanger-tol/curationpretext](docs/images/curationpretext-dark.png#gh-dark-mode-only)
 
+[![GitHub Actions CI Status](https://github.com/sanger-tol/curationpretext/workflows/nf-core%20CI/badge.svg)](https://github.com/sanger-tol/curationpretext/actions?query=workflow%3A%22nf-core+CI%22)
+[![GitHub Actions Linting Status](https://github.com/sanger-tol/curationpretext/workflows/nf-core%20linting/badge.svg)](https://github.com/sanger-tol/curationpretext/actions?query=workflow%3A%22nf-core+linting%22)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.12773958-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.12773958)
+
 [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
@@ -111,6 +114,8 @@ sanger-tol/curationpretext was originally written by Damon-Lee B Pointon (@DLBPo
 
 We thank the following people for their extensive assistance in the development of this pipeline:
 
+- @muffato - For reviews.
+
 - @yumisims - TreeVal and Software.
 
 - @weaglesBio - TreeVal and Software.
@@ -125,8 +130,7 @@ For further information or help, don't hesitate to get in touch on the [Slack `#
 
 ## Citations
 
-<!-- TODO nf-core: Add citation for pipeline after first release. Uncomment lines below and update Zenodo doi and badge at the top of this file. -->
-<!-- If you use  sanger-tol/curationpretext for your analysis, please cite it using the following doi: [10.5281/zenodo.XXXXXX](https://doi.org/10.5281/zenodo.XXXXXX) -->
+If you use sanger-tol/curationpretext for your analysis, please cite it using the following doi: [10.5281/zenodo.12773958](https://doi.org/10.5281/zenodo.12773958)
 
 An extensive list of references for the tools used by the pipeline can be found in the [`CITATIONS.md`](CITATIONS.md) file.
 

bin/get_avgcov.sh

@@ -6,9 +6,10 @@
 # into bed format for use
 # -------------------
 # Author = yy5
+# Modified = dp24
 # -------------------
 version='1.0.0'
-if [ $1 == '-v'];
+if [ $1 == '-v' ];
 then
     echo "$version"
 else

conf/local_testing/test.config

@@ -0,0 +1,28 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run sanger-tol/curationpretext -profile test,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'GitHub Test profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus    = 20
+    max_memory  = '30.GB'
+    max_time    = '6.h'
+
+    input       = "/nfs/treeoflife-01/teams/tola/users/dp24/curationpretext/TreeValTinyData/assembly/draft/grTriPseu1.fa"
+    outdir      = "./results"
+    longread    = "/nfs/treeoflife-01/teams/tola/users/dp24/curationpretext/TreeValTinyData/genomic_data/pacbio/"
+    cram        = "/nfs/treeoflife-01/teams/tola/users/dp24/curationpretext/TreeValTinyData/genomic_data/hic-arima/"
+    sample      = "CurationPretextTest"
+    teloseq     = "TTAGGG"
+}

conf/modules.config

@@ -133,7 +133,7 @@ process {
     }
 
     withName: PRETEXTMAP_STANDRD {
-        ext.args        = { "${meta.map_order.equals("length") ? "--sortby length": "--sortby nosort" } --mapq 0 --memory ${task.memory.giga}" }
+        ext.args        = { "${meta.map_order.equals("length") ? "--sortby length": "--sortby nosort" } --mapq 0" }
         ext.prefix      = { "${meta.id}_normal_pi" }
     }
 

conf/test_full.config

@@ -12,15 +12,16 @@
 
 cleanup = true
 
+executor.cpus = 50
+executor.memory = 50.GB
+executor.time = 1.h
+
 params {
     config_profile_name        = 'Full test profile'
     config_profile_description = 'Full test dataset to check pipeline function'
 
     // Input data for full size test
     // Limit resources so that this can run on GitHub Actions
-    max_cpus    = 4
-    max_memory  = '10.GB'
-    max_time    = '6.h'
 
     sample      = "testing"
     input       = "/lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValTinyData/assembly/draft/grTriPseu1.fa"

lib/Utils.groovy

@@ -22,7 +22,7 @@ class Utils {
 
         // Check that all channels are present
         // This channel list is ordered by required channel priority.
-        def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults']
+        def required_channels_in_order = ['conda-forge', 'bioconda']
         def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean
 
         // Check that they are in the right order