diff --git a/.github/include.yaml b/.github/include.yaml new file mode 100644 index 00000000..a3629f4c --- /dev/null +++ b/.github/include.yaml @@ -0,0 +1,10 @@ +".": + - ./.github/workflows/** + - ./nf-test.config + - ./nextflow.config +tests: + - ./assets/* + - ./bin/* + - ./conf/* + - ./main.nf + - ./nextflow_schema.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 583783db..f920d3b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,88 +1,106 @@ name: nf-core CI # This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: - push: - branches: - - dev pull_request: - release: - types: [published] - workflow_dispatch: env: NXF_ANSI_LOG: false + NFT_VER: "0.9.0" + NFT_WORKDIR: "~" + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true - jobs: - test: - name: Run pipeline with test data - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'Plant-Food-Research-Open/assemblyqc') }}" + nf-test-changes: + name: Check for changes runs-on: ubuntu-latest - strategy: - matrix: - NXF_VER: - - "24.04.2" - TEST_PARAMS: - - minimal - - invalid - - stub - - noltr - - hicparam - include: - - OPTION_STUB: "" - - OPTION_STUB: "-stub" - TEST_PARAMS: stub - - OPTION_STUB: "-stub" - TEST_PARAMS: hicparam + outputs: + nf_test_files: ${{ steps.list.outputs.components }} steps: - - name: Check out pipeline code - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 + - uses: actions/checkout@v4.2.1 with: - version: ${{ matrix.NXF_VER }} + fetch-depth: 0 - - name: Disk space cleanup - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + - name: List nf-test files + id: list + uses: adamrtalbot/detect-nf-test-changes@v0.0.4 + with: + head: ${{ github.sha }} + base: origin/${{ github.base_ref }} + include: .github/include.yaml - - name: Run pipeline with test data + - name: print list of nf-test files run: | - nextflow run \ - ${GITHUB_WORKSPACE} \ - -profile test,docker \ - -params-file ./tests/${{ matrix.TEST_PARAMS }}/params.json \ - ${{ matrix.OPTION_STUB }} \ - --outdir ./results - - nf-test: - name: Run nf-tests - # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'Plant-Food-Research-Open/assemblyqc') }}" + echo ${{ steps.list.outputs.components }} + + test: + name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + needs: [nf-test-changes] + if: needs.nf-test-changes.outputs.nf_test_files != '[]' runs-on: ubuntu-latest strategy: + fail-fast: false matrix: NXF_VER: - "24.04.2" + + nf_test_files: ["${{ fromJson(needs.nf-test-changes.outputs.nf_test_files) }}"] + profile: + - "docker" + steps: - name: Check out pipeline code - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + uses: actions/checkout@v4.2.1 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 with: - version: ${{ matrix.NXF_VER }} + version: "${{ matrix.NXF_VER }}" + + - uses: actions/setup-python@v5.2.0 + with: + python-version: "3.11" + architecture: "x64" + + - name: Install pdiff to see diff between nf-test snapshots + run: | + python -m pip install --upgrade pip + pip install pdiff + + - uses: nf-core/setup-nf-test@v1.1.2 + + - name: Run nf-test + run: | + nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --tap=test.tap + + - uses: pcolby/tap-summary@v1.1.1 + with: + path: >- + test.tap - - name: Install nf-test - uses: nf-core/setup-nf-test@v1.1.2 + - name: Output log on failure + if: failure() + run: | + sudo apt install bat > /dev/null + batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log + + confirm-pass: + runs-on: ubuntu-latest + needs: [test] + if: always() + steps: + - name: All tests ok + if: ${{ !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 - - name: Run nf-tests + - name: debug-print + if: always() run: | - nf-test \ - test \ - --verbose \ - tests + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 99704fe9..749d7926 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 1. Added Gfastats [#126](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/126) 2. Updated nf-core/template to 3.0.1 [#149](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/149) 3. Updated `samtools faidx` to 1.21 +4. Now using nf-test for pipeline level testing [#153](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/153) ### `Fixed` diff --git a/nf-test.config b/nf-test.config index 9d4198ff..042da10d 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,8 +1,10 @@ config { + testsDir "." + workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" + configFile "tests/nextflow.config" - testsDir "tests" - workDir ".nf-test" - configFile "nextflow.config" - profile "" - + plugins { + load "nft-bam@0.4.0" + load "nft-utils@0.0.3" + } } diff --git a/tests/README.md b/tests/README.md index 257defe3..6e935db0 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,14 +14,17 @@ Or using [singularity](https://docs.sylabs.io/guides/3.0/user-guide/installation nextflow run plant-food-research-open/assemblyqc -r main -profile singularity,test --outdir results ``` -## Local Testing +## nf-test and Continuous Integration (CI) -The test sets included in this directory can be executed by first downloading the pipeline from GitHub and then executing the following command: +The GitHub [CI action](../.github/workflows/ci.yml) included with the pipeline continuously tests the pipeline using [nf-test](https://www.nf-test.com). -```bash -./main.nf -profile docker -params-file tests/minimal/params.json --outdir results -``` +## Testing Merqury Datasets -## Continuous Integration (CI) +Three Merqury datasets are included here which can be tested by pointing to one of the parameters file. -The GitHub [CI action](../.github/workflows/ci.yml) included with the pipeline continuously tests the pipeline with the various test sets listed in this directory. +```bash +./main.nf \ + -profile \ + -params-file tests/merqury//params.json \ + --outdir results +``` diff --git a/tests/hicparam/assemblysheet.csv b/tests/hicparam/assemblysheet.csv index 19e256c0..222ecb0b 100644 --- a/tests/hicparam/assemblysheet.csv +++ b/tests/hicparam/assemblysheet.csv @@ -1,2 +1,2 @@ tag,fasta -test,tests/hicparam/test_genome.fa.gz +test,https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/sarscov2/genome/genome.fasta diff --git a/tests/hicparam/main.nf.test b/tests/hicparam/main.nf.test new file mode 100644 index 00000000..064dc0c6 --- /dev/null +++ b/tests/hicparam/main.nf.test @@ -0,0 +1,35 @@ +nextflow_pipeline { + + name "Test with hic param" + script "main.nf" + + test("hic param - stub") { + + options '-stub' + + when { + params { + input = "$baseDir/tests/hicparam/assemblysheet.csv" + hic = "$baseDir/tests/hicparam/hic/Dummy_hic_{1,2}.merged.fq.gz" + outdir = "$outputDir" + } + } + + then { + def stable_path = getAllFilesFromDir(params.outdir, false, ['pipeline_info/*.{html,json,txt,yml}', 'report.{html,json}'], null, ['**']) + + assertAll( + { assert workflow.success}, + { assert snapshot( + [ + 'successful tasks': workflow.trace.succeeded().size(), + 'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"), + 'stable paths': stable_path + ] + ).match() } + ) + } + + } + +} diff --git a/tests/hicparam/main.nf.test.snap b/tests/hicparam/main.nf.test.snap new file mode 100644 index 00000000..6ca72d89 --- /dev/null +++ b/tests/hicparam/main.nf.test.snap @@ -0,0 +1,102 @@ +{ + "hic param - stub": { + "content": [ + { + "successful tasks": 21, + "versions": { + "AGP2ASSEMBLY": { + "juicebox_scripts": "0.1.0" + }, + "ASSEMBLATHON_STATS": { + "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2" + }, + "ASSEMBLY2BEDPE": { + "python": "3.11.3", + "pandas": "2.1.1" + }, + "BWA_INDEX": { + "bwa": "0.7.18-r1243-dirty" + }, + "BWA_MEM": { + "bwa": "0.7.18-r1243-dirty", + "samtools": 1.2 + }, + "FASTAVALIDATOR": { + "py_fasta_validator": 0.6 + }, + "FASTP": { + "fastp": "0.23.4" + }, + "FASTQC_RAW": { + "fastqc": "0.12.1" + }, + "FASTQC_TRIM": { + "fastqc": "0.12.1" + }, + "HIC2HTML": { + "python": "3.11.3" + }, + "HICQC": { + "hic_qc.py": "0+untagged.261.g6881c33" + }, + "JUICER_SORT": { + "sort": 8.3 + }, + "MAKEAGPFROMFASTA": { + "juicebox_scripts": "0.1.0" + }, + "MATLOCK_BAM2_JUICER": { + "matlock": 20181227 + }, + "RUNASSEMBLYVISUALIZER": { + "run-assembly-visualizer.sh": "18 July 2016" + }, + "SAMBLASTER": { + "samblaster": "0.1.26", + "samtools": "1.19.2" + }, + "SAMTOOLS_FAIDX": { + "samtools": 1.21 + }, + "SEQKIT_RMDUP": { + "seqkit": "v2.8.0" + }, + "SEQKIT_SORT": { + "seqkit": "v2.8.0" + }, + "TAG_ASSEMBLY": { + "pigz": "2.3.4" + }, + "Workflow": { + "plant-food-research-open/assemblyqc": "v2.2.0+dev" + } + }, + "stable paths": [ + "test_stats.csv:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.agp.assembly:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.assembly.bedpe:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.fastp.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.fastp.json:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.fastp.log:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.paired.fail.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "Dummy_hic_1.fail.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "Dummy_hic_1.fastp.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "Dummy_hic_2.fail.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "Dummy_hic_2.fastp.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "Dummy_hic.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.zip:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.zip:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.on.test.pdf:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.hic:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.html:md5,bbd8f07f11522eb75bb9429e86e95713" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-10T16:11:00.660108" + } +} \ No newline at end of file diff --git a/tests/hicparam/params.json b/tests/hicparam/params.json deleted file mode 100644 index c99d6c4b..00000000 --- a/tests/hicparam/params.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "config_profile_name": "Test to verify hic param validation", - "config_profile_description": "Test to verify hic param validation", - "input": "tests/hicparam/assemblysheet.csv", - "hic": "tests/hicparam/hic/Dummy_hic_{1,2}.merged.fq.gz" -} diff --git a/tests/hicparam/test_genome.fa.gz b/tests/hicparam/test_genome.fa.gz deleted file mode 100644 index 1a720c1f..00000000 Binary files a/tests/hicparam/test_genome.fa.gz and /dev/null differ diff --git a/tests/invalid/assemblysheet.csv b/tests/invalid/assemblysheet.csv index 93e81420..bfee9e8d 100644 --- a/tests/invalid/assemblysheet.csv +++ b/tests/invalid/assemblysheet.csv @@ -1,5 +1,5 @@ tag,fasta,gff3 FI1,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.fna.gz,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.gff.gz TT_2021a,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/021/950/295/GCA_021950295.1_ASM2195029v1/GCA_021950295.1_ASM2195029v1_genomic.fna.gz,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.gff.gz -MISC,tests/invalid/invalid.fsa.gz -DUPSEQ,tests/invalid/dupseq.fsa.gz +MISC,https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/tests/invalid/invalid.fsa.gz +DUPSEQ,https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/tests/invalid/dupseq.fsa.gz diff --git a/tests/invalid/main.nf.test b/tests/invalid/main.nf.test new file mode 100644 index 00000000..93cb3310 --- /dev/null +++ b/tests/invalid/main.nf.test @@ -0,0 +1,35 @@ +nextflow_pipeline { + + name "Test with invalid input files" + script "main.nf" + + test("invalid") { + + when { + params { + input = "$baseDir/tests/invalid/assemblysheet.csv" + outdir = "$outputDir" + } + } + + then { + def stable_path = getAllFilesFromDir(params.outdir, false, ['pipeline_info/*.{html,json,txt,yml}', 'report.{html,json}'], null, ['**']) + + assertAll( + { assert workflow.success}, + { assert 'WARN: GFF3 validation failed for TT_2021a' in workflow.stdout }, + { assert 'WARN: FASTA validation failed for MISC' in workflow.stdout }, + { assert 'WARN: FASTA validation failed for DUPSEQ due to presence of duplicate sequences' in workflow.stdout }, + { assert snapshot( + [ + 'successful tasks': workflow.trace.succeeded().size(), + 'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"), + 'stable paths': stable_path + ] + ).match() } + ) + } + + } + +} diff --git a/tests/invalid/main.nf.test.snap b/tests/invalid/main.nf.test.snap new file mode 100644 index 00000000..bdd1665d --- /dev/null +++ b/tests/invalid/main.nf.test.snap @@ -0,0 +1,54 @@ +{ + "invalid": { + "content": [ + { + "successful tasks": 25, + "versions": { + "ASSEMBLATHON_STATS": { + "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2" + }, + "FASTAVALIDATOR": { + "py_fasta_validator": 0.6 + }, + "GT_GFF3": { + "genometools": "1.6.5" + }, + "GT_GFF3VALIDATOR": { + "genometools": "1.6.5" + }, + "GT_STAT": { + "genometools": "1.6.5" + }, + "GUNZIP_FASTA": { + "gunzip": 1.1 + }, + "GUNZIP_GFF3": { + "gunzip": 1.1 + }, + "SAMTOOLS_FAIDX": { + "samtools": 1.21 + }, + "SEQKIT_RMDUP": { + "seqkit": "v2.8.0" + }, + "TAG_ASSEMBLY": { + "pigz": "2.3.4" + }, + "Workflow": { + "plant-food-research-open/assemblyqc": "v2.2.0+dev" + } + }, + "stable paths": [ + "FI1_stats.csv:md5,8d1274e52117e39b413ff71a76c40331", + "TT_2021a_stats.csv:md5,55fa0923a6d47fdd19201486848b48fe", + "FI1.gt.stat.yml:md5,2fc1b9c84af0c2323d78a9ac623a3022" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-10T15:59:12.150838" + } +} \ No newline at end of file diff --git a/tests/invalid/params.json b/tests/invalid/params.json deleted file mode 100644 index eb2dab01..00000000 --- a/tests/invalid/params.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "config_profile_name": "Invalid profile", - "config_profile_description": "Profile to test invalid files", - "input": "tests/invalid/assemblysheet.csv" -} diff --git a/tests/merqury/phased2x.mp/assemblysheet.csv.local b/tests/merqury/phased2x.mp/assemblysheet.csv.local deleted file mode 100644 index f90aec6f..00000000 --- a/tests/merqury/phased2x.mp/assemblysheet.csv.local +++ /dev/null @@ -1,4 +0,0 @@ -tag,fasta,reads_1,reads_2,maternal_reads_1,paternal_reads_1 -COL,https://gembox.cbcb.umd.edu/triobinning/athal_COL.fasta,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/SRR3703081_1.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/SRR3703081_2.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/athal_CVI.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/athal_COL.fastq.gz -CVI,https://gembox.cbcb.umd.edu/triobinning/athal_CVI.fasta,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/SRR3703081_1.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/SRR3703081_2.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/athal_CVI.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/athal_COL.fastq.gz -FI1,https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/003/814/445/GCA_003814445.1_ASM381444v1/GCA_003814445.1_ASM381444v1_genomic.fna.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/SRR8238189_1.fastq.gz,/Users/hrauxr/Projects/test-data/assemblyqc/merqury.fk/phased2x.mp/SRR8238189_2.fastq.gz diff --git a/tests/minimal/main.nf.test b/tests/minimal/main.nf.test new file mode 100644 index 00000000..b4dae9be --- /dev/null +++ b/tests/minimal/main.nf.test @@ -0,0 +1,32 @@ +nextflow_pipeline { + + name "Test with minimal input" + script "main.nf" + + test("minimal") { + + when { + params { + input = "$baseDir/assets/assemblysheetv2.csv" + outdir = "$outputDir" + } + } + + then { + def stable_path = getAllFilesFromDir(params.outdir, false, ['pipeline_info/*.{html,json,txt,yml}', 'report.{html,json}'], null, ['**']) + + assertAll( + { assert workflow.success}, + { assert snapshot( + [ + 'successful tasks': workflow.trace.succeeded().size(), + 'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"), + 'stable paths': stable_path + ] + ).match() } + ) + } + + } + +} diff --git a/tests/minimal/main.nf.test.snap b/tests/minimal/main.nf.test.snap new file mode 100644 index 00000000..7538396f --- /dev/null +++ b/tests/minimal/main.nf.test.snap @@ -0,0 +1,53 @@ +{ + "minimal": { + "content": [ + { + "successful tasks": 11, + "versions": { + "ASSEMBLATHON_STATS": { + "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2" + }, + "FASTAVALIDATOR": { + "py_fasta_validator": 0.6 + }, + "GT_GFF3": { + "genometools": "1.6.5" + }, + "GT_GFF3VALIDATOR": { + "genometools": "1.6.5" + }, + "GT_STAT": { + "genometools": "1.6.5" + }, + "GUNZIP_FASTA": { + "gunzip": 1.1 + }, + "GUNZIP_GFF3": { + "gunzip": 1.1 + }, + "SAMTOOLS_FAIDX": { + "samtools": 1.21 + }, + "SEQKIT_RMDUP": { + "seqkit": "v2.8.0" + }, + "TAG_ASSEMBLY": { + "pigz": "2.3.4" + }, + "Workflow": { + "plant-food-research-open/assemblyqc": "v2.2.0+dev" + } + }, + "stable paths": [ + "FI1_stats.csv:md5,8d1274e52117e39b413ff71a76c40331", + "FI1.gt.stat.yml:md5,2fc1b9c84af0c2323d78a9ac623a3022" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-10T14:37:54.336611" + } +} diff --git a/tests/minimal/params.json b/tests/minimal/params.json deleted file mode 100644 index 0b8c5117..00000000 --- a/tests/minimal/params.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "config_profile_name": "Test profile", - "config_profile_description": "Minimal test dataset to check pipeline function", - "input": "https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/assets/assemblysheetv2.csv" -} diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 00000000..1d5de542 --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,22 @@ +/* +======================================================================================== + Nextflow config file for running tests +======================================================================================== +*/ + +params { + modules_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/modules/' +} + +timeline { enabled = false } +report { enabled = false } +trace { enabled = false } +dag { enabled = false } + +process { + resourceLimits = [ + cpus: 4, + memory: '15.GB', + time: '1.h' + ] +} diff --git a/tests/noltr/main.nf.test b/tests/noltr/main.nf.test new file mode 100644 index 00000000..98f9e1ce --- /dev/null +++ b/tests/noltr/main.nf.test @@ -0,0 +1,55 @@ +nextflow_pipeline { + + name "Test with a genome which does not have LTRs" + script "main.nf" + + test("noltr") { + + when { + params { + input = "$baseDir/tests/noltr/assemblysheet.csv" + lai_skip = false + outdir = "$outputDir" + } + } + + then { + def stable_path = getAllFilesFromDir( + params.outdir, + false, + [ + 'pipeline_info/*.{html,json,txt,yml}', + 'report.{html,json}', + 'lai/*.LAI.log', + 'lai/*.restored.ids.gff3' + ], + null, + ['**'] + ) + + def stable_name = getAllFilesFromDir( + params.outdir, + true, + [ + 'pipeline_info/*.{html,json,txt,yml}' + ], + null, + ['**'] + ) + + assertAll( + { assert workflow.success}, + { assert snapshot( + [ + 'successful tasks': workflow.trace.succeeded().size(), + 'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"), + 'stable paths': stable_path, + 'stable names': getRelativePath(stable_name, outputDir), + ] + ).match() } + ) + } + + } + +} diff --git a/tests/noltr/main.nf.test.snap b/tests/noltr/main.nf.test.snap new file mode 100644 index 00000000..c336d8a5 --- /dev/null +++ b/tests/noltr/main.nf.test.snap @@ -0,0 +1,89 @@ +{ + "noltr": { + "content": [ + { + "successful tasks": 26, + "versions": { + "ASSEMBLATHON_STATS": { + "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2" + }, + "CAT_CAT": { + "pigz": "2.3.4" + }, + "CUSTOM_RESTOREGFFIDS": { + "python": "3.10.2" + }, + "CUSTOM_SHORTENFASTAIDS": { + "python": "3.8.13", + "biopython": 1.75 + }, + "FASTAVALIDATOR": { + "py_fasta_validator": 0.6 + }, + "GUNZIP_FASTA": { + "gunzip": 1.1 + }, + "LTRFINDER": { + "LTR_FINDER_parallel": "v1.1", + "ltr_finder": "v1.07" + }, + "LTRHARVEST": { + "LTR_HARVEST_parallel": "v1.1", + "genometools": "1.6.5" + }, + "LTRRETRIEVER_LAI": { + "lai": "beta3.2" + }, + "LTRRETRIEVER_LTRRETRIEVER": { + "LTR_retriever": "v2.9.9" + }, + "SAMTOOLS_FAIDX": { + "samtools": 1.21 + }, + "SEQKIT_RMDUP": { + "seqkit": "v2.8.0" + }, + "TAG_ASSEMBLY": { + "pigz": "2.3.4" + }, + "UNMASK_IF_ANY": { + "seqkit": "v2.8.0" + }, + "Workflow": { + "plant-food-research-open/assemblyqc": "v2.2.0+dev" + } + }, + "stable paths": [ + "FI1_stats.csv:md5,8d1274e52117e39b413ff71a76c40331", + "sarscov2_stats.csv:md5,9ec4147b73cae550a38337520fc028dd", + "FI1.LAI.out:md5,c958c2f4187e0ae42060d304556e0e6a", + "FI1.LTRlib.fa:md5,e7cc7ee7269dd5a09773cfbedba0de06", + "FI1.short.ids.tsv:md5,948c19c1ccd05463a1f5d23f0615b169", + "sarscov2.short.ids.tsv:md5,d7a2af88e8549586e5616bff6a88bd71" + ], + "stable names": [ + "assemblathon_stats", + "assemblathon_stats/FI1_stats.csv", + "assemblathon_stats/sarscov2_stats.csv", + "lai", + "lai/FI1.LAI.log", + "lai/FI1.LAI.out", + "lai/FI1.LTRlib.fa", + "lai/FI1.restored.ids.gff3", + "lai/FI1.short.ids.tsv", + "lai/sarscov2.short.ids.tsv", + "pipeline_info", + "report.html", + "report.json", + "synteny", + "synteny/plotsr" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-10T15:40:52.594249" + } +} \ No newline at end of file diff --git a/tests/noltr/params.json b/tests/noltr/params.json deleted file mode 100644 index e52470db..00000000 --- a/tests/noltr/params.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "config_profile_name": "No LTRs assembly profile", - "config_profile_description": "Profile to test an assembly without LTRs", - "input": "tests/noltr/assemblysheet.csv", - "lai_skip": false -} diff --git a/tests/stub/main.nf.test b/tests/stub/main.nf.test new file mode 100644 index 00000000..528ec8b5 --- /dev/null +++ b/tests/stub/main.nf.test @@ -0,0 +1,55 @@ +nextflow_pipeline { + + name "Test the entire pipeline in stub mode" + script "main.nf" + + test("full - stub") { + options '-stub' + + when { + params { + input = "$baseDir/assets/assemblysheetv2.csv" + gfastats_skip = false + ncbi_fcs_adaptor_skip = false + ncbi_fcs_adaptor_empire = "euk" + ncbi_fcs_gx_skip = false + ncbi_fcs_gx_tax_id = 12 + ncbi_fcs_gx_db_path = "$baseDir/tests/stub/gxdb/test" + busco_skip = false + busco_mode = "genome" + busco_lineage_datasets = "fungi_odb10 hypocreales_odb10" + tidk_skip = false + tidk_repeat_seq = "TTTGGG" + lai_skip = false + kraken2_skip = false + kraken2_db_path = "$baseDir/tests/stub/kraken2/k2_minusb_20231009.tar.gz" + hic = "$baseDir/tests/stub/hic/Dummy_hic.R{1,2}.fq.gz" + hic_skip_fastp = true + hic_skip_fastqc = false + synteny_skip = false + synteny_mummer_skip = false + synteny_plotsr_skip = false + synteny_xref_assemblies = "$baseDir/assets/xrefsheet.csv" + merqury_skip = false + outdir = "$outputDir" + } + } + + then { + def stable_path = getAllFilesFromDir(params.outdir, false, ['pipeline_info/*.{html,json,txt,yml}', 'report.{html,json}'], null, ['**']) + + assertAll( + { assert workflow.success}, + { assert snapshot( + [ + 'successful tasks': workflow.trace.succeeded().size(), + 'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"), + 'stable paths': stable_path + ] + ).match() } + ) + } + + } + +} diff --git a/tests/stub/main.nf.test.snap b/tests/stub/main.nf.test.snap new file mode 100644 index 00000000..257765b7 --- /dev/null +++ b/tests/stub/main.nf.test.snap @@ -0,0 +1,307 @@ +{ + "full - stub": { + "content": [ + { + "successful tasks": 91, + "versions": { + "AGP2ASSEMBLY": { + "juicebox_scripts": "0.1.0" + }, + "ASSEMBLATHON_STATS": { + "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2" + }, + "ASSEMBLY2BEDPE": { + "python": "3.11.3", + "pandas": "2.1.1" + }, + "BUNDLELINKS": { + "python": "3.11.3" + }, + "BUSCO_ANNOTATION": { + "busco": "5.7.1" + }, + "BUSCO_ASSEMBLY": { + "busco": "5.7.1" + }, + "BWA_INDEX": { + "bwa": "0.7.18-r1243-dirty" + }, + "BWA_MEM": { + "bwa": "0.7.18-r1243-dirty", + "samtools": 1.2 + }, + "CAT_CAT": { + "pigz": "2.3.4" + }, + "CIRCOS": { + "circos": "v0.69-8", + "perl": 5.032001 + }, + "COLOURBUNDLELINKS": { + "python": "3.11.3", + "perl": "5.32.1" + }, + "CUSTOM_RELABELFASTA": { + "python": "3.8.13", + "biopython": 1.75 + }, + "CUSTOM_SHORTENFASTAIDS": { + "python": "3.8.13", + "biopython": 1.75 + }, + "CUSTOM_SRATOOLSNCBISETTINGS": { + "sratools": "3.0.8" + }, + "DNADIFF": { + "dnadiff": 1.3 + }, + "EXTRACT_PROTEINS": { + "gffread": "0.12.7" + }, + "FASTAVALIDATOR": { + "py_fasta_validator": 0.6 + }, + "FASTQC_RAW": { + "fastqc": "0.12.1" + }, + "FCS_FCSADAPTOR": { + "FCS-adaptor": "0.5.0" + }, + "FILTERSORTFASTA": { + "samtools": "1.16.1" + }, + "FILTER_BY_LENGTH": { + "seqkit": "v2.8.0" + }, + "GENERATEKARYOTYPE": { + "awk": "1.3.4 20200120", + "grep": "(GNU grep) 3.4", + "sed": "(GNU sed) 4.7" + }, + "GETFASTALENGTH": { + "samtools": "1.16.1" + }, + "GFASTATS": { + "gfastats": "1.3.6" + }, + "GT_GFF3": { + "genometools": "1.6.5" + }, + "GT_GFF3VALIDATOR": { + "genometools": "1.6.5" + }, + "GT_STAT": { + "genometools": "1.6.5" + }, + "GUNZIP_FASTA": { + "gunzip": 1.1 + }, + "GUNZIP_GFF3": { + "gunzip": 1.1 + }, + "HIC2HTML": { + "python": "3.11.3" + }, + "HICQC": { + "hic_qc.py": "0+untagged.261.g6881c33" + }, + "JUICER_SORT": { + "sort": 8.3 + }, + "KRAKEN2": { + "kraken2": "2.1.2" + }, + "KRAKEN2_KRONA_PLOT": { + "KronaTools": "2.7.1" + }, + "LINEARSYNTENY": { + "python": "3.11.3", + "pandas": "2.1.1", + "plotly": "5.20.0" + }, + "LTRFINDER": { + "LTR_FINDER_parallel": "v1.1", + "ltr_finder": "v1.07" + }, + "LTRHARVEST": { + "LTR_HARVEST_parallel": "v1.1", + "genometools": "1.6.5" + }, + "LTRRETRIEVER_LAI": { + "lai": "beta3.2" + }, + "LTRRETRIEVER_LTRRETRIEVER": { + "LTR_retriever": "v2.9.9" + }, + "MAKEAGPFROMFASTA": { + "juicebox_scripts": "0.1.0" + }, + "MATLOCK_BAM2_JUICER": { + "matlock": 20181227 + }, + "MERQURY_MERQURY": { + "merqury": 1.3 + }, + "MERYL_COUNT": { + "meryl": "1.4.1" + }, + "MERYL_UNIONSUM": { + "meryl": "1.4.1" + }, + "MINIMAP2_ALIGN": { + "minimap2": "2.28-r1209" + }, + "MUMMER": { + "nucmer": "4.0.0rc1" + }, + "NCBI_FCS_GX_KRONA_PLOT": { + "KronaTools": "2.7.1" + }, + "NCBI_FCS_GX_SCREEN_SAMPLES": { + "fcs_gx": "0.5.4" + }, + "NCBI_FCS_GX_SETUP_SAMPLE": { + "ubuntu": "20.04.6l" + }, + "PLOTSR": { + "plotsr": "1.1.1" + }, + "RELABELBUNDLELINKS": { + "python": "3.11.3" + }, + "RELABELFASTALENGTH": { + "python": "3.11.3" + }, + "RUNASSEMBLYVISUALIZER": { + "run-assembly-visualizer.sh": "18 July 2016" + }, + "SAMBLASTER": { + "samblaster": "0.1.26", + "samtools": "1.19.2" + }, + "SAMTOOLS_FAIDX": { + "samtools": 1.21 + }, + "SEQKIT_RMDUP": { + "seqkit": "v2.8.0" + }, + "SEQKIT_SORT": { + "seqkit": "v2.8.0" + }, + "SORT_BY_LENGTH": { + "seqkit": "v2.8.0" + }, + "SPLITBUNDLEFILE": { + "awk": "1.3.4 20200120" + }, + "SRATOOLS_FASTERQDUMP": { + "sratools": "3.0.8", + "pigz": 2.6 + }, + "SRATOOLS_PREFETCH": { + "sratools": "3.1.0", + "curl": "8.5.0" + }, + "SYRI": { + "syri": "1.7.0" + }, + "TAG_ASSEMBLY": { + "pigz": "2.3.4" + }, + "TIDK_EXPLORE": { + "tidk": "0.2.41" + }, + "TIDK_PLOT_APOSTERIORI": { + "tidk": "0.2.41" + }, + "TIDK_PLOT_APRIORI": { + "tidk": "0.2.41" + }, + "TIDK_SEARCH_APOSTERIORI": { + "tidk": "0.2.41" + }, + "TIDK_SEARCH_APRIORI": { + "tidk": "0.2.41" + }, + "UNMASK_IF_ANY": { + "seqkit": "v2.8.0" + }, + "UNTAR": { + "untar": 1.34 + }, + "Workflow": { + "plant-food-research-open/assemblyqc": "v2.2.0+dev" + } + }, + "stable paths": [ + "FI1_stats.csv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.gt.stat.yml:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.assembly_summary:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.hic:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.html:md5,6bd2e97a3a2199609c2f2a5d4d314b1e", + "FI1.agp.assembly:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.assembly.bedpe:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.R.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.R.zip:md5,d41d8cd98f00b204e9800998ecf8427e", + "Dummy_hic.R.on.FI1.pdf:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.kraken2.cut:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.kraken2.krona.cut:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.kraken2.krona.html:md5,a629f1aab2e8dce3e921119498d669fd", + "FI1.kraken2.report:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.LAI.log:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.LAI.out:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.LTRlib.fa:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.short.ids.tsv:md5,fcf920d9a7b57a1e3c29a9e88673330f", + "FI1.FI1.qv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.completeness.stats:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.dist_only.hist:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.hist.ploidy:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.qv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-asm.fl.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-asm.hist:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-asm.ln.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-asm.st.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-cn.fl.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-cn.hist:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-cn.ln.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.spectra-cn.st.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1_only.bed:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1_only.wig:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.fcs_adaptor_report.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.fcs.gx.krona.cut:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.fcs.gx.krona.html:md5,4e24c1bdb5d3e2666e647390a9ececce", + "FI1.fcs_gx_report.txt:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.inter.tax.rpt.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.taxonomy.rpt:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.on.JAD.all.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.on.JAD.all.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.on.JAD.all.svg:md5,d41d8cd98f00b204e9800998ecf8427e", + "bundled.links.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "circos.conf:md5,d41d8cd98f00b204e9800998ecf8427e", + "karyotype.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.on.TT_2021a.all.html:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.on.TT_2021a.all.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.on.TT_2021a.all.svg:md5,d41d8cd98f00b204e9800998ecf8427e", + "bundled.links.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "circos.conf:md5,d41d8cd98f00b204e9800998ecf8427e", + "karyotype.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.plotsr.csv:md5,d2e982f8f4fb8e32dc2f0ae74e688aef", + "JAD.plotsr.csv:md5,1c9f46f53f33361b80e126cc5d410c7d", + "TT_2021a.plotsr.csv:md5,dba0dd1f7c25e345052363030deace6b", + "plotsr.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.aposteriori.svg:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.aposteriori.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.apriori.svg:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.apriori.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.tidk.explore.tsv:md5,d41d8cd98f00b204e9800998ecf8427e", + "FI1.top.sequence.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-10T14:56:44.730644" + } +} \ No newline at end of file diff --git a/tests/stub/params.json b/tests/stub/params.json deleted file mode 100644 index 85bdf0ce..00000000 --- a/tests/stub/params.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "config_profile_name": "Full stub test", - "config_profile_description": "Full test of the pipeline in stub mode", - "input": "assets/assemblysheetv2.csv", - "gfastats_skip": false, - "ncbi_fcs_adaptor_skip": false, - "ncbi_fcs_adaptor_empire": "euk", - "ncbi_fcs_gx_skip": false, - "ncbi_fcs_gx_tax_id": 12, - "ncbi_fcs_gx_db_path": "tests/stub/gxdb/test", - "busco_skip": false, - "busco_mode": "genome", - "busco_lineage_datasets": "fungi_odb10 hypocreales_odb10", - "tidk_skip": false, - "tidk_repeat_seq": "TTTGGG", - "lai_skip": false, - "kraken2_skip": false, - "kraken2_db_path": "tests/stub/kraken2/k2_minusb_20231009.tar.gz", - "hic": "tests/stub/hic/Dummy_hic.R{1,2}.fq.gz", - "hic_skip_fastp": true, - "hic_skip_fastqc": false, - "synteny_skip": false, - "synteny_mummer_skip": false, - "synteny_plotsr_skip": false, - "synteny_xref_assemblies": "assets/xrefsheet.csv", - "merqury_skip": false -} diff --git a/tests/tiny/main.nf.test b/tests/tiny/main.nf.test new file mode 100644 index 00000000..c3254e9f --- /dev/null +++ b/tests/tiny/main.nf.test @@ -0,0 +1,32 @@ +nextflow_pipeline { + + name "Test with a tiny input genome" + script "main.nf" + + test("tiny") { + + when { + params { + input = "$baseDir/tests/tiny/assemblysheet.csv" + outdir = "$outputDir" + } + } + + then { + def stable_path = getAllFilesFromDir(params.outdir, false, ['pipeline_info/*.{html,json,txt,yml}', 'report.{html,json}'], null, ['**']) + + assertAll( + { assert workflow.success}, + { assert snapshot( + [ + 'successful tasks': workflow.trace.succeeded().size(), + 'versions': removeNextflowVersion("$outputDir/pipeline_info/software_versions.yml"), + 'stable paths': stable_path + ] + ).match() } + ) + } + + } + +} diff --git a/tests/tiny/main.nf.test.snap b/tests/tiny/main.nf.test.snap new file mode 100644 index 00000000..c3a241a3 --- /dev/null +++ b/tests/tiny/main.nf.test.snap @@ -0,0 +1,47 @@ +{ + "tiny": { + "content": [ + { + "successful tasks": 9, + "versions": { + "ASSEMBLATHON_STATS": { + "assemblathon_stats": "github/PlantandFoodResearch/assemblathon2-analysis/a93cba2" + }, + "FASTAVALIDATOR": { + "py_fasta_validator": 0.6 + }, + "GT_GFF3": { + "genometools": "1.6.5" + }, + "GT_GFF3VALIDATOR": { + "genometools": "1.6.5" + }, + "GT_STAT": { + "genometools": "1.6.5" + }, + "SAMTOOLS_FAIDX": { + "samtools": 1.21 + }, + "SEQKIT_RMDUP": { + "seqkit": "v2.8.0" + }, + "TAG_ASSEMBLY": { + "pigz": "2.3.4" + }, + "Workflow": { + "plant-food-research-open/assemblyqc": "v2.2.0+dev" + } + }, + "stable paths": [ + "sarscov2_stats.csv:md5,9ec4147b73cae550a38337520fc028dd", + "sarscov2.gt.stat.yml:md5,2504b472be03bdebaa033b0e507e4bfe" + ] + } + ], + "meta": { + "nf-test": "0.9.0", + "nextflow": "24.04.4" + }, + "timestamp": "2024-10-10T14:46:59.600343" + } +} \ No newline at end of file diff --git a/tests/tiny/params.json b/tests/tiny/params.json deleted file mode 100644 index 71262048..00000000 --- a/tests/tiny/params.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "config_profile_name": "Test profile", - "config_profile_description": "Tiny test dataset to check pipeline function", - "input": "tests/tiny/assemblysheet.csv" -}