diff --git a/.editorconfig b/.editorconfig index 44cb1a23..87e16fad 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,7 @@ indent_style = space [*.{md,yml,yaml,html,css,scss,js,cff}] indent_size = 2 + # These files are edited and tested upstream in nf-core/modules [/modules/nf-core/**] charset = unset @@ -42,6 +43,7 @@ indent_style = unset [/assets/email*] indent_size = unset + # ignore python and markdown [*.{py,md}] indent_style = unset @@ -49,3 +51,4 @@ indent_style = unset # ignore perl [*.{pl,pm}] indent_size = unset +indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 79b1aa06..fca1a245 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -16,7 +16,7 @@ If you'd like to write some code for plant-food-research-open/assemblyqc, the st 1. Check that there isn't already an issue about your idea in the [plant-food-research-open/assemblyqc issues](https://github.com/plant-food-research-open/assemblyqc/issues) to avoid duplicating work. If there isn't one already, please create one so that others know you're working on this 2. [Fork](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) the [plant-food-research-open/assemblyqc repository](https://github.com/plant-food-research-open/assemblyqc) to your GitHub account 3. Make the necessary changes / additions within your forked repository following [Pipeline conventions](#pipeline-contribution-conventions) -4. Use `nf-core schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). +4. Use `nf-core pipelines schema build` and add any new parameters to the pipeline JSON schema (requires [nf-core tools](https://github.com/nf-core/tools) >= 1.10). 5. Submit a Pull Request against the `dev` branch and wait for the code to be reviewed and merged If you're not used to this workflow with git, you can start with some [docs from GitHub](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests) or even their [excellent `git` resources](https://try.github.io/). @@ -37,7 +37,7 @@ There are typically two types of tests that run: ### Lint tests `nf-core` has a [set of guidelines](https://nf-co.re/developers/guidelines) which all pipelines must adhere to. -To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core lint ` command. +To enforce these and ensure that all pipelines stay in sync, we have developed a helper tool which runs checks on the pipeline code. This is in the [nf-core/tools repository](https://github.com/nf-core/tools) and once installed can be run locally with the `nf-core pipelines lint ` command. If any failures or warnings are encountered, please follow the listed URL for more documentation. @@ -68,17 +68,17 @@ If you wish to contribute a new step, please use the following coding standards: 2. Write the process block (see below). 3. Define the output channel if needed (see below). 4. Add any new parameters to `nextflow.config` with a default (see below). -5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core schema build` tool). +5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core pipelines schema build` tool). 6. Add sanity checks and validation for all relevant parameters. 7. Perform local tests to validate that the new code works as expected. 8. If applicable, add a new test command in `.github/workflow/ci.yml`. -9. Add a description of the output files and if relevant any appropriate images from the MultiQC report to `docs/output.md`. +9. Add a description of the output files and if relevant any appropriate images from the report to `docs/output.md`. ### Default values Parameters should be initialised / defined with default values in `nextflow.config` under the `params` scope. -Once there, use `nf-core schema build` to add to `nextflow_schema.json`. +Once there, use `nf-core pipelines schema build` to add to `nextflow_schema.json`. ### Default processes resource requirements @@ -95,7 +95,7 @@ Please use the following naming schemes, to make it easy to understand what is g ### Nextflow version bumping -If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core bump-version --nextflow . [min-nf-version]` +If you are using a new feature from core Nextflow, you may bump the minimum required version of nextflow in the pipeline with: `nf-core pipelines bump-version --nextflow . [min-nf-version]` ### Images and figures diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index af436aa6..e3135746 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -9,46 +9,34 @@ body: description: A clear and concise description of what the bug is. validations: required: true + - type: textarea id: command_used attributes: label: Command used and terminal output - description: Steps to reproduce the behaviour. Please paste the command you used - to launch the pipeline and the output from your terminal. + description: Steps to reproduce the behaviour. Please paste the command you used to launch the pipeline and the output from your terminal. render: console - placeholder: "$ nextflow run ... - + placeholder: | + $ nextflow run ... Some output where something broke - " - type: textarea id: files attributes: label: Relevant files - description: "Please drag and drop the relevant files here. Create a `.zip` archive - if the extension is not allowed. - - Your verbose log file `.nextflow.log` is often useful _(this is a hidden file - in the directory where you launched the pipeline)_ as well as custom Nextflow - configuration files. + description: | + Please drag and drop the relevant files here. Create a `.zip` archive if the extension is not allowed. + Your verbose log file `.nextflow.log` is often useful _(this is a hidden file in the directory where you launched the pipeline)_ as well as custom Nextflow configuration files. - " - type: textarea id: system attributes: label: System information - description: "* Nextflow version _(eg. 23.04.0)_ - + description: | + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ - * Executor _(eg. slurm, local, awsbatch)_ - - * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, - or Apptainer)_ - + * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ - * Version of plant-food-research-open/assemblyqc _(eg. 1.1, 1.5, 1.8.2)_ - - " diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0cd29f73..3104f8b2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/plant-food-r - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/plant-food-research-open/assemblyqc/tree/main/.github/CONTRIBUTING.md) -- [ ] Make sure your code lints (`nf-core lint`). +- [ ] Make sure your code lints (`nf-core pipelines lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index c5f090a7..81a9eaac 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -11,7 +11,7 @@ jobs: steps: # PRs to the nf-core repo main branch are only ok if coming from the nf-core repo `dev` or any `patch` branches - name: Check PRs - if: github.repository == 'plant-food-research-open/assemblyqc' + if: github.repository == 'Plant-Food-Research-Open/assemblyqc' run: | { [[ ${{ github.event.pull_request.head.repo.full_name }} == Plant-Food-Research-Open/assemblyqc ]] && [[ $GITHUB_HEAD_REF == "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c0a86ca..583783db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: pull_request: release: types: [published] + workflow_dispatch: env: NXF_ANSI_LOG: false @@ -19,12 +20,12 @@ jobs: test: name: Run pipeline with test data # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'plant-food-research-open/assemblyqc') }}" + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'Plant-Food-Research-Open/assemblyqc') }}" runs-on: ubuntu-latest strategy: matrix: NXF_VER: - - "23.04.0" + - "24.04.2" TEST_PARAMS: - minimal - invalid @@ -53,21 +54,20 @@ jobs: run: | nextflow run \ ${GITHUB_WORKSPACE} \ - --outdir ./results \ - -profile docker \ - -params-file \ - ./tests/${{ matrix.TEST_PARAMS }}/params.json \ - ${{ matrix.OPTION_STUB }} + -profile test,docker \ + -params-file ./tests/${{ matrix.TEST_PARAMS }}/params.json \ + ${{ matrix.OPTION_STUB }} \ + --outdir ./results nf-test: name: Run nf-tests # Only run on push if this is the nf-core dev branch (merged PRs) - if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'plant-food-research-open/assemblyqc') }}" + if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'Plant-Food-Research-Open/assemblyqc') }}" runs-on: ubuntu-latest strategy: matrix: NXF_VER: - - "23.04.0" + - "24.04.2" steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 4a2e3eb4..7cc2c387 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -1,4 +1,4 @@ -name: Test successful pipeline download with 'nf-core download' +name: Test successful pipeline download with 'nf-core pipelines download' # Run the workflow when: # - dispatched manually @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: testbranch: - description: "The specific branch you wish to utilize for the test execution of nf-core download." + description: "The specific branch you wish to utilize for the test execution of nf-core pipelines download." required: true default: "dev" pull_request: @@ -39,9 +39,11 @@ jobs: with: python-version: "3.12" architecture: "x64" - - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 + + - name: Setup Apptainer + uses: eWaterCycle/setup-apptainer@4bb22c52d4f63406c49e94c804632975787312b3 # v2.0.0 with: - singularity-version: 3.8.3 + apptainer-version: 1.3.4 - name: Install dependencies run: | @@ -54,33 +56,64 @@ jobs: echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} + - name: Make a cache directory for the container images + run: | + mkdir -p ./singularity_container_images + - name: Download the pipeline env: - NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images run: | - nf-core download ${{ env.REPO_LOWERCASE }} \ + nf-core pipelines download ${{ env.REPO_LOWERCASE }} \ --revision ${{ env.REPO_BRANCH }} \ --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ --compress "none" \ --container-system 'singularity' \ - --container-library "quay.io" -l "docker.io" -l "ghcr.io" \ + --container-library "quay.io" -l "docker.io" -l "community.wave.seqera.io" \ --container-cache-utilisation 'amend' \ - --download-configuration + --download-configuration 'yes' - name: Inspect download run: tree ./${{ env.REPOTITLE_LOWERCASE }} + - name: Count the downloaded number of container images + id: count_initial + run: | + image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) + echo "Initial container image count: $image_count" + echo "IMAGE_COUNT_INITIAL=$image_count" >> ${GITHUB_ENV} + - name: Run the downloaded pipeline (stub) id: stub_run_pipeline continue-on-error: true env: - NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results - name: Run the downloaded pipeline (stub run not supported) id: run_pipeline if: ${{ job.steps.stub_run_pipeline.status == failure() }} env: - NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_CACHEDIR: ./singularity_container_images NXF_SINGULARITY_HOME_MOUNT: true run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -profile test,singularity --outdir ./results + + - name: Count the downloaded number of container images + id: count_afterwards + run: | + image_count=$(ls -1 ./singularity_container_images | wc -l | xargs) + echo "Post-pipeline run container image count: $image_count" + echo "IMAGE_COUNT_AFTER=$image_count" >> ${GITHUB_ENV} + + - name: Compare container image counts + run: | + if [ "${{ env.IMAGE_COUNT_INITIAL }}" -ne "${{ env.IMAGE_COUNT_AFTER }}" ]; then + initial_count=${{ env.IMAGE_COUNT_INITIAL }} + final_count=${{ env.IMAGE_COUNT_AFTER }} + difference=$((final_count - initial_count)) + echo "$difference additional container images were \n downloaded at runtime . The pipeline has no support for offline runs!" + tree ./singularity_container_images + exit 1 + else + echo "The pipeline can be downloaded successfully!" + fi diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 31ca9786..f1e67583 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -9,7 +9,7 @@ jobs: if: > contains(github.event.comment.html_url, '/pull/') && contains(github.event.comment.body, '@nf-core-bot fix linting') && - github.repository == 'plant-food-research-open/assemblyqc' + github.repository == 'Plant-Food-Research-Open/assemblyqc' runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 1fcafe88..6bfe9373 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,6 +1,6 @@ name: nf-core linting # This workflow is triggered on pushes and PRs to the repository. -# It runs the `nf-core lint` and markdown lint tests to ensure +# It runs the `nf-core pipelines lint` and markdown lint tests to ensure # that the code meets the nf-core guidelines. on: push: @@ -41,17 +41,32 @@ jobs: python-version: "3.12" architecture: "x64" + - name: read .nf-core.yml + uses: pietrobolcato/action-read-yaml@1.1.0 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yml + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install nf-core + pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + + - name: Run nf-core pipelines lint + if: ${{ github.base_ref != 'main' }} + env: + GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} + run: nf-core -l lint_log.txt pipelines lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - - name: Run nf-core lint + - name: Run nf-core pipelines lint --release + if: ${{ github.base_ref == 'main' }} env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_PR_COMMIT: ${{ github.event.pull_request.head.sha }} - run: nf-core -l lint_log.txt lint --dir ${GITHUB_WORKSPACE} --markdown lint_results.md + run: nf-core -l lint_log.txt pipelines lint --release --dir ${GITHUB_WORKSPACE} --markdown lint_results.md - name: Save PR number if: ${{ always() }} diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 40acc23f..42e519bf 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 + uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 # v6 with: workflow: linting.yml workflow_conclusion: completed diff --git a/.github/workflows/template_version_comment.yml b/.github/workflows/template_version_comment.yml new file mode 100644 index 00000000..9dea41f0 --- /dev/null +++ b/.github/workflows/template_version_comment.yml @@ -0,0 +1,43 @@ +name: nf-core template version comment +# This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version. +# It posts a comment to the PR, even if it comes from a fork. + +on: pull_request_target + +jobs: + template_version: + runs-on: ubuntu-latest + steps: + - name: Check out pipeline code + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + + - name: Read template version from .nf-core.yml + uses: pietrobolcato/action-read-yaml@1.0.0 + id: read_yml + with: + config: ${{ github.workspace }}/.nf-core.yml + + - name: Install nf-core + run: | + python -m pip install --upgrade pip + pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }} + + - name: Check nf-core outdated + id: nf_core_outdated + run: pip list --outdated | grep nf-core + + - name: Post nf-core template version comment + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 + if: | + ${{ steps.nf_core_outdated.outputs.stdout }} =~ 'nf-core' + with: + repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} + allow-repeats: false + message: | + ## :warning: Newer version of the nf-core template is available. + + Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}. + Please update your pipeline to the latest version. + + For more documentation on how to update your pipeline, please see the [nf-core documentation](https://github.com/nf-core/tools?tab=readme-ov-file#sync-a-pipeline-with-the-template) and [Synchronisation documentation](https://nf-co.re/docs/contributing/sync). + # diff --git a/.gitpod.yml b/.gitpod.yml index 105a1821..46118637 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,17 +4,14 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update - - name: unset JAVA_TOOL_OPTIONS - command: | - unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack - - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code + #- esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code - charliermarsh.ruff # Code linter Ruff diff --git a/.nf-core.yml b/.nf-core.yml index 0ade11e9..a021ce5d 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,5 +1,9 @@ +bump_version: null lint: files_exist: + - conf/igenomes.config + - conf/igenomes_ignored.config + - assets/multiqc_config.yml - CODE_OF_CONDUCT.md - assets/nf-core-assemblyqc_logo_light.png - docs/images/nf-core-assemblyqc_logo_light.png @@ -7,21 +11,37 @@ lint: - .github/ISSUE_TEMPLATE/config.yml - .github/workflows/awstest.yml - .github/workflows/awsfulltest.yml - - assets/multiqc_config.yml - - conf/igenomes.config files_unchanged: - - docs/README.md - - .github/PULL_REQUEST_TEMPLATE.md + - LICENSE - .github/CONTRIBUTING.md + - .github/PULL_REQUEST_TEMPLATE.md - .github/workflows/branch.yml - - LICENSE + - .github/workflows/linting.yml + - docs/README.md + multiqc_config: false + template_strings: false nextflow_config: - manifest.name - manifest.homePage - multiqc_config: False - template_strings: False -nf_core_version: 2.14.1 + - validation.help.beforeText + - validation.help.afterText + - validation.summary.beforeText + - validation.summary.afterText +nf_core_version: 3.0.0 +org_path: null repository_type: pipeline template: - prefix: plant-food-research-open - skip: [] + author: Usman Rashid, Ken Smith, Ross Crowhurst, Chen Wu, Marcus Davy + description: A NextFlow pipeline which evaluates assembly quality with multiple + QC tools and presents the results in a unified html report. + force: true + is_nfcore: false + name: assemblyqc + org: plant-food-research-open + outdir: . + skip_features: + - igenomes + - multiqc + - fastqc + version: 3.0.0 +update: null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1168e07f..5875f2b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "2.7.3" + rev: "3.0.3" hooks: - id: editorconfig-checker alias: ec diff --git a/.prettierignore b/.prettierignore index 437d763d..610e5069 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ + email_template.html adaptivecard.json slackreport.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 436cacf8..b7ff96cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,23 +3,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.2.0+dev - [01-Oct-2024] +## v2.2.0+dev - [09-Oct-2024] ### `Added` 1. Added Gfastats [#126](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/126) +2. Updated nf-core/template to 3.0.0 [#149](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/149) ### `Fixed` ### `Dependencies` 1. Nextflow!>=23.04.0 -2. nf-validation@1.1.3 +2. nf-schema@2.1.1 ### `Deprecated` 1. Reduced the GenomeTools stats figures to 300 DPI [#142](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/142) 2. Now `synteny_mummer_min_bundle_size` is set to `1000000` by default [#142](https://github.com/Plant-Food-Research-Open/assemblyqc/issues/142) +3. `results` is not the default output directory anymore +4. Removed a number of unnecessary parameters: `monochromeLogs`, `config_profile_contact`, `config_profile_url`, `validationFailUnrecognisedParams`, `validationLenientMode`, `validationSchemaIgnoreParams`, `validationShowHiddenParams` `validate_params` +5. Resource parameters have been removed: `max_memory`, `max_cpus`, `max_time` ## v2.1.1 - [20-Sep-2024] diff --git a/README.md b/README.md index 7adcae92..9b47b55b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ +# plant-food-research-open/assemblyqc + [![GitHub Actions CI Status](https://github.com/plant-food-research-open/assemblyqc/actions/workflows/ci.yml/badge.svg)](https://github.com/plant-food-research-open/assemblyqc/actions/workflows/ci.yml) [![GitHub Actions Linting Status](https://github.com/plant-food-research-open/assemblyqc/actions/workflows/linting.yml/badge.svg)](https://github.com/plant-food-research-open/assemblyqc/actions/workflows/linting.yml)[![Cite Article](http://img.shields.io/badge/DOI-10.1093/bioinformatics/btae477-1073c8?labelColor=000000)](https://doi.org/10.1093/bioinformatics/btae477) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.2-23aa62.svg)](https://www.nextflow.io/) [![run with conda ❌](http://img.shields.io/badge/run%20with-conda%20❌-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) @@ -106,8 +108,7 @@ nextflow run plant-food-research-open/assemblyqc \ ``` > [!WARNING] -> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; -> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files). ### Plant&Food Users diff --git a/assets/schema_input.json b/assets/schema_input.json index afec8fe7..83f4915d 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -1,11 +1,12 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/main/assets/schema_input.json", "title": "plant-food-research-open/assemblyqc pipeline - params.input schema", "description": "Schema for the file provided with params.input", "type": "array", "items": { "type": "object", + "uniqueEntries": ["tag"], "properties": { "tag": { "type": "string", @@ -72,8 +73,7 @@ "type": "string", "maxLength": 0 } - ], - "dependentRequired": ["reads_1"] + ] }, "maternal_reads_1": { "errorMessage": "maternal_reads_1 should be a SRA ID for paired FASTQ files or FASTX file path without spaces and must have extension '.f(a|asta|as|sa|na|astq|q)' or '.f(a|asta|as|sa|na|astq|q).gz'", @@ -86,8 +86,7 @@ "type": "string", "maxLength": 0 } - ], - "dependentRequired": ["reads_1", "paternal_reads_1"] + ] }, "maternal_reads_2": { "errorMessage": "FASTX file path cannot contain spaces and must have extension '.f(a|asta|as|sa|na|astq|q)' or '.f(a|asta|as|sa|na|astq|q).gz'", @@ -100,8 +99,7 @@ "type": "string", "maxLength": 0 } - ], - "dependentRequired": ["maternal_reads_1"] + ] }, "paternal_reads_1": { "errorMessage": "paternal_reads_1 should be a SRA ID for paired FASTQ files or FASTX file path without spaces and must have extension '.f(a|asta|as|sa|na|astq|q)' or '.f(a|asta|as|sa|na|astq|q).gz'", @@ -114,8 +112,7 @@ "type": "string", "maxLength": 0 } - ], - "dependentRequired": ["reads_1", "maternal_reads_1"] + ] }, "paternal_reads_2": { "errorMessage": "FASTX file path cannot contain spaces and must have extension '.f(a|asta|as|sa|na|astq|q)' or '.f(a|asta|as|sa|na|astq|q).gz'", @@ -128,10 +125,16 @@ "type": "string", "maxLength": 0 } - ], - "dependentRequired": ["paternal_reads_1"] + ] } }, - "required": ["tag", "fasta"] + "required": ["tag", "fasta"], + "dependentRequired": { + "reads_2": ["reads_1"], + "maternal_reads_1": ["reads_1", "paternal_reads_1"], + "maternal_reads_2": ["maternal_reads_1"], + "paternal_reads_1": ["reads_1", "maternal_reads_1"], + "paternal_reads_2": ["paternal_reads_1"] + } } } diff --git a/assets/schema_xref_assemblies.json b/assets/schema_xref_assemblies.json index 0da6ca1c..9a82cfbf 100644 --- a/assets/schema_xref_assemblies.json +++ b/assets/schema_xref_assemblies.json @@ -1,11 +1,12 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/main/assets/schema_xref_assemblies.json", "title": "plant-food-research-open/assemblyqc pipeline - params.synteny_xref_assemblies schema", "description": "Schema for the file provided with params.synteny_xref_assemblies", "type": "array", "items": { "type": "object", + "uniqueEntries": ["tag"], "properties": { "tag": { "type": "string", diff --git a/conf/base.config b/conf/base.config index 87a5d686..20473408 100644 --- a/conf/base.config +++ b/conf/base.config @@ -10,9 +10,9 @@ process { - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 * task.attempt } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 @@ -24,30 +24,30 @@ process { // If possible, it would be nice to keep the same label naming convention when // adding in your local modules too. withLabel:process_single { - cpus = { check_max( 1 , 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 } + memory = { 6.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 2 * task.attempt } + memory = { 12.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_medium { - cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + cpus = { 6 * task.attempt } + memory = { 36.GB * task.attempt } + time = { 8.h * task.attempt } } withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 72.GB * task.attempt, 'memory' ) } - time = { check_max( 16.h * task.attempt, 'time' ) } + cpus = { 12 * task.attempt } + memory = { 72.GB * task.attempt } + time = { 16.h * task.attempt } } withLabel:process_long { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withLabel:process_high_memory { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + memory = { 200.GB * task.attempt } } withLabel:error_ignore { errorStrategy = 'ignore' @@ -57,23 +57,23 @@ process { maxRetries = 2 } withName:NCBI_FCS_GX_SCREEN_SAMPLES { - time = { check_max( 20.h * task.attempt, 'time' ) } - memory = { check_max( 512.GB * task.attempt, 'memory' ) } + time = { 20.h * task.attempt } + memory = { 512.GB * task.attempt } } withName:KRAKEN2 { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + memory = { 200.GB * task.attempt } } withName:BWA_MEM { - time = { check_max( 2.day * task.attempt, 'time' ) } + time = { 2.day * task.attempt } } withName:SAMBLASTER { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withName:DNADIFF { - time = { check_max( 7.day * task.attempt, 'time' ) } + time = { 7.day * task.attempt } } withName:MERQURY_HAPMERS { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withName:CREATEREPORT { cache = false diff --git a/conf/test.config b/conf/test.config index b48e728f..2417719d 100644 --- a/conf/test.config +++ b/conf/test.config @@ -10,14 +10,17 @@ ---------------------------------------------------------------------------------------- */ +process { + resourceLimits = [ + cpus: 4, + memory: '15.GB', + time: '1.h' + ] +} + params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' input = 'https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/assets/assemblysheetv2.csv' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' } diff --git a/conf/test_full.config b/conf/test_full.config index 14f95cb9..2eadbf9b 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -10,6 +10,14 @@ ---------------------------------------------------------------------------------------- */ +process { + resourceLimits = [ + cpus: 10, + memory: '32.GB', + time: '6.h' + ] +} + params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/docs/parameters.md b/docs/parameters.md index c9d7eefb..a6b2e912 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -4,11 +4,11 @@ A Nextflow pipeline which evaluates assembly quality with multiple QC tools and ## Input/output options -| Parameter | Description | Type | Default | Required | Hidden | -| --------- | ------------------------------------------------------------------------------------------------------------------------ | -------- | --------- | -------- | ------ | -| `input` | Input assembly sheet in CSV format | `string` | | True | | -| `outdir` | The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure. | `string` | ./results | True | | -| `email` | Email address for completion summary. | `string` | | | | +| Parameter | Description | Type | Default | Required | Hidden | +| --------- | ------------------------------------------------------------------------------------------------------------------------ | -------- | ------- | -------- | ------ | +| `input` | Input assembly sheet in CSV format | `string` | | True | | +| `outdir` | The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure. | `string` | | True | | +| `email` | Email address for completion summary. | `string` | | | | ## Validation options @@ -99,16 +99,6 @@ A Nextflow pipeline which evaluates assembly quality with multiple QC tools and | `merqury_skip` | Skip merqury analysis | `boolean` | True | | | | `merqury_kmer_length` | kmer length for merqury analysis | `integer` | 21 | | | -## Max job request options - -Set the top limit for requested resources for any single job. - -| Parameter | Description | Type | Default | Required | Hidden | -| ------------ | ---------------------------------------------------------------------------------- | --------- | ------- | -------- | ------ | -| `max_cpus` | Maximum number of CPUs that can be requested for any single job. | `integer` | 16 | | True | -| `max_memory` | Maximum amount of memory that can be requested for any single job. Example: '8.GB' | `string` | 512.GB | | True | -| `max_time` | Maximum amount of time that can be requested for any single job. Example: '1.day' | `string` | 7.day | | True | - ## Institutional config options Parameters used to describe centralised config profiles. These should not be edited. @@ -119,24 +109,16 @@ Parameters used to describe centralised config profiles. These should not be edi | `custom_config_base` | Base directory for Institutional configs. | `string` | https://raw.githubusercontent.com/nf-core/configs/master | | True | | `config_profile_name` | Institutional config name. | `string` | | | True | | `config_profile_description` | Institutional config description. | `string` | | | True | -| `config_profile_contact` | Institutional config contact information. | `string` | | | True | -| `config_profile_url` | Institutional config URL link. | `string` | | | True | ## Generic options Less common options for the pipeline, typically set in a config file. -| Parameter | Description | Type | Default | Required | Hidden | -| ---------------------------------- | ----------------------------------------------------------------------- | --------- | ------- | -------- | ------ | -| `help` | Display help text. | `boolean` | | | True | -| `version` | Display version and exit. | `boolean` | | | True | -| `publish_dir_mode` | Method used to save pipeline results to output directory. | `string` | copy | | True | -| `email_on_fail` | Email address for completion summary, only when pipeline fails. | `string` | | | True | -| `plaintext_email` | Send plain-text email instead of HTML. | `boolean` | | | True | -| `monochrome_logs` | Do not use coloured log outputs. | `boolean` | | | True | -| `monochromeLogs` | Do not use coloured log outputs. | `boolean` | | | True | -| `hook_url` | Incoming hook URL for messaging service | `string` | | | True | -| `validate_params` | Boolean whether to validate parameters against the schema at runtime | `boolean` | True | | True | -| `validationShowHiddenParams` | Show all params when using `--help` | `boolean` | | | True | -| `validationFailUnrecognisedParams` | Validation of parameters fails when an unrecognised parameter is found. | `boolean` | | | True | -| `validationLenientMode` | Validation of parameters in lenient more. | `boolean` | | | True | +| Parameter | Description | Type | Default | Required | Hidden | +| ------------------ | --------------------------------------------------------------- | --------- | ------- | -------- | ------ | +| `version` | Display version and exit. | `boolean` | | | True | +| `publish_dir_mode` | Method used to save pipeline results to output directory. | `string` | copy | | True | +| `email_on_fail` | Email address for completion summary, only when pipeline fails. | `string` | | | True | +| `plaintext_email` | Send plain-text email instead of HTML. | `boolean` | | | True | +| `monochrome_logs` | Do not use coloured log outputs. | `boolean` | | | True | +| `hook_url` | Incoming hook URL for messaging service | `string` | | | True | diff --git a/docs/usage.md b/docs/usage.md index 509fdbd2..20f10823 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -26,7 +26,6 @@ - [Custom Containers](#custom-containers) - [Custom Tool Arguments](#custom-tool-arguments) - [nf-core/configs](#nf-coreconfigs) -- [Azure Resource Requests](#azure-resource-requests) - [Running in the background](#running-in-the-background) - [Nextflow memory requirements](#nextflow-memory-requirements) @@ -127,7 +126,7 @@ The data for these examples comes from: [umd.edu](https://obj.umiacs.umd.edu/mar ## Minimum System Requirements -All the modules have been tested to work on a single machine with 10 CPUs + 30 GBs of memory, except NCBI FCS GX and Kraken2. Their minimum requirements are: +All the modules have been tested to work on a single machine with 10 CPUs + 32 GBs of memory, except NCBI FCS GX and Kraken2. Their minimum requirements are: - NCBI FCS GX: 1 CPU + 512 GBs memory - Kraken2: 1 CPU + 200 GBs memory @@ -164,9 +163,9 @@ The above pipeline run specified with a params file in yaml format: nextflow run plant-food-research-open/assemblyqc -profile docker -params-file params.yaml ``` -with `params.yaml` containing: +with: -```yaml +```yaml title="params.yaml" input: "./assemblysheet.csv" outdir: "./results/" ``` @@ -187,7 +186,7 @@ It is a good idea to specify a pipeline version when running the pipeline on you First, go to the [plant-food-research-open/assemblyqc releases page](https://github.com/plant-food-research-open/assemblyqc/releases) and find the latest pipeline version - numeric only (eg. `1.3.1`). Then specify this when running the pipeline with `-r` (one hyphen) - eg. `-r 1.3.1`. Of course, you can switch to another version by changing the number after the `-r` flag. -This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. For example, at the bottom of the MultiQC reports. +This version number will be logged in reports when you run the pipeline, so that you'll know what you used when you look back in the future. To further assist in reproducbility, you can use share and re-use [parameter files](#running-the-pipeline) to repeat pipeline runs with the same settings without having to write out a command with every single parameter. @@ -273,14 +272,6 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). -## Azure Resource Requests - -To be used with the `azurebatch` profile by specifying the `-profile azurebatch`. -We recommend providing a compute `params.vm_type` of `Standard_D16_v3` VMs by default but these options can be changed if required. - -Note that the choice of VM size depends on your quota and the overall workload during the analysis. -For a thorough list, please refer the [Azure Sizes for virtual machines in Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes). - ## Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. diff --git a/local_assemblyqc b/local_assemblyqc index 858acea8..bcfff8c3 100755 --- a/local_assemblyqc +++ b/local_assemblyqc @@ -17,10 +17,9 @@ nextflow run \ -profile docker,test_full \ -resume \ $stub \ - --max_cpus 8 \ - --max_memory '32.GB' \ --ncbi_fcs_gx_skip false \ --ncbi_fcs_gx_db_path ../dbs/gxdb/test \ --busco_download_path ../dbs/busco \ --kraken2_skip false \ - --kraken2_db_path ../dbs/kraken2db/k2_minusb + --kraken2_db_path ../dbs/kraken2db/k2_minusb \ + --outdir results diff --git a/main.nf b/main.nf index 91e82dff..7e46ba85 100755 --- a/main.nf +++ b/main.nf @@ -7,8 +7,6 @@ ---------------------------------------------------------------------------------------- */ -nextflow.enable.dsl = 2 - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS @@ -18,7 +16,6 @@ nextflow.enable.dsl = 2 include { ASSEMBLYQC } from './workflows/assemblyqc' include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_assemblyqc_pipeline' include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_assemblyqc_pipeline' - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ NAMED WORKFLOWS FOR PIPELINE @@ -54,7 +51,6 @@ workflow PLANTFOODRESEARCHOPEN_ASSEMBLYQC { ch_params_as_json, ch_summary_params_as_json ) - } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -65,14 +61,11 @@ workflow PLANTFOODRESEARCHOPEN_ASSEMBLYQC { workflow { main: - // // SUBWORKFLOW: Run initialisation tasks // PIPELINE_INITIALISATION ( params.version, - params.help, - params.validate_params, params.monochrome_logs, args, params.outdir, @@ -92,7 +85,6 @@ workflow { PIPELINE_INITIALISATION.out.params_as_json, PIPELINE_INITIALISATION.out.summary_params_as_json ) - // // SUBWORKFLOW: Run completion tasks // @@ -102,7 +94,8 @@ workflow { params.plaintext_email, params.outdir, params.monochrome_logs, - params.hook_url + params.hook_url, + ) } diff --git a/modules.json b/modules.json index 21d052a3..37e0d8eb 100644 --- a/modules.json +++ b/modules.json @@ -279,9 +279,9 @@ "git_sha": "2fdce49d30c0254f76bc0f13c55c17455c1251ab", "installed_by": ["subworkflows"] }, - "utils_nfvalidation_plugin": { + "utils_nfschema_plugin": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "bbd5a41f4535a8defafe6080e00ea74c45f4f96c", "installed_by": ["subworkflows"] } } diff --git a/nextflow.config b/nextflow.config index aafebe96..b0065cc4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -74,22 +74,15 @@ params { merqury_kmer_length = 21 // Output options - outdir = './results' + outdir = null email = null - // Max resource options - max_memory = '512.GB' - max_cpus = 16 - max_time = '7.day' - // Boilerplate options publish_dir_mode = 'copy' email_on_fail = null plaintext_email = false monochrome_logs = false - monochromeLogs = false hook_url = null - help = false version = false // Config options @@ -97,34 +90,20 @@ params { config_profile_description = null custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" - config_profile_contact = null - config_profile_url = null - - // Schema validation default options - validationFailUnrecognisedParams = false - validationLenientMode = false - validationSchemaIgnoreParams = '' - validationShowHiddenParams = false - validate_params = true +} +// Max resources +process { + resourceLimits = [ + cpus: 16, + memory: '512.GB', + time: '7.day' + ] } // Load base.config by default for all pipelines includeConfig 'conf/base.config' -// Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} - -// Load plant-food-research-open/assemblyqc custom profiles from different institutions. -// try { -// includeConfig "${params.custom_config_base}/pipeline/assemblyqc.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/assemblyqc profiles: ${params.custom_config_base}/pipeline/assemblyqc.config") -// } profiles { debug { dumpHashes = true @@ -139,7 +118,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - conda.channels = ['conda-forge', 'bioconda', 'defaults'] + conda.channels = ['conda-forge', 'bioconda'] apptainer.enabled = false } mamba { @@ -229,18 +208,19 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } -// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile -// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled -// Set to your registry if you have a mirror of containers -apptainer.registry = 'quay.io' -docker.registry = 'quay.io' -podman.registry = 'quay.io' -singularity.registry = 'quay.io' +// Load nf-core custom profiles from different Institutions +includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/nfcore_custom.config" : "/dev/null" -// Nextflow plugins -plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet -} +// Load plant-food-research-open/assemblyqc custom profiles from different institutions. +// includeConfig !System.getenv('NXF_OFFLINE') && params.custom_config_base ? "${params.custom_config_base}/pipeline/assemblyqc.config" : "/dev/null" +// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile +// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled +// Set to your registry if you have a mirror of containers +apptainer.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' +singularity.registry = 'quay.io' +charliecloud.registry = 'quay.io' // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. @@ -253,8 +233,15 @@ env { JULIA_DEPOT_PATH = "/usr/local/share/julia" } -// Capture exit codes from upstream processes when piping -process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Set bash options +process.shell = """\ +bash + +set -e # Exit if a tool returns a non-zero status/exit code +set -u # Treat unset variables and parameters as an error +set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute +set -C # No clobber - prevent output redirection from overwriting files. +""" // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false @@ -283,43 +270,27 @@ manifest { homePage = 'https://github.com/plant-food-research-open/assemblyqc' description = """A Nextflow pipeline which evaluates assembly quality with multiple QC tools and presents the results in a unified html report.""" mainScript = 'main.nf' - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=24.04.2' version = '2.2.0+dev' doi = 'https://doi.org/10.1093/bioinformatics/btae477' } -// Load modules.config for DSL2 module specific options -includeConfig 'conf/modules.config' +// Nextflow plugins +plugins { + id 'nf-schema@2.1.1' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} -// Function to ensure that resource requirements don't go beyond -// a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } +validation { + defaultIgnoreParams = ["genomes"] + help { + enabled = true + command = "nextflow run $manifest.name -profile --input assemblysheet.csv --outdir " + fullParameter = "help_full" + showHiddenParameter = "show_hidden" } + monochromeLogs = params.monochrome_logs } + +// Load modules.config for DSL2 module specific options +includeConfig 'conf/modules.config' + diff --git a/nextflow_schema.json b/nextflow_schema.json index 3c3fb30d..eda14c7a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/main/nextflow_schema.json", "title": "plant-food-research-open/assemblyqc pipeline parameters", "description": "A Nextflow pipeline which evaluates assembly quality with multiple QC tools and presents the results in a unified html report.", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -25,8 +25,7 @@ "type": "string", "format": "directory-path", "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", - "fa_icon": "fas fa-folder-open", - "default": "./results" + "fa_icon": "fas fa-folder-open" }, "email": { "type": "string", @@ -349,38 +348,6 @@ } } }, - "max_job_request_options": { - "title": "Max job request options", - "type": "object", - "fa_icon": "fab fa-acquisitions-incorporated", - "description": "Set the top limit for requested resources for any single job.", - "help_text": "If you are running on a smaller system, a pipeline step requesting more resources than are available may cause the Nextflow to stop the run with an error. These options allow you to cap the maximum resources requested by any single job so that the pipeline will run on your system.\n\nNote that you can not _increase_ the resources requested by any job using these options. For that you will need your own configuration file. See [the nf-core website](https://nf-co.re/usage/configuration) for details.", - "properties": { - "max_cpus": { - "type": "integer", - "description": "Maximum number of CPUs that can be requested for any single job.", - "default": 16, - "fa_icon": "fas fa-microchip", - "hidden": true - }, - "max_memory": { - "type": "string", - "description": "Maximum amount of memory that can be requested for any single job. Example: '8.GB'", - "default": "512.GB", - "fa_icon": "fas fa-memory", - "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", - "hidden": true - }, - "max_time": { - "type": "string", - "description": "Maximum amount of time that can be requested for any single job. Example: '1.day'", - "default": "7.day", - "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", - "hidden": true - } - } - }, "institutional_config_options": { "title": "Institutional config options", "type": "object", @@ -413,18 +380,6 @@ "description": "Institutional config description.", "hidden": true, "fa_icon": "fas fa-users-cog" - }, - "config_profile_contact": { - "type": "string", - "description": "Institutional config contact information.", - "hidden": true, - "fa_icon": "fas fa-users-cog" - }, - "config_profile_url": { - "type": "string", - "description": "Institutional config URL link.", - "hidden": true, - "fa_icon": "fas fa-users-cog" } } }, @@ -435,12 +390,6 @@ "description": "Less common options for the pipeline, typically set in a config file.", "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", "properties": { - "help": { - "type": "boolean", - "description": "Display help text.", - "fa_icon": "fas fa-question-circle", - "hidden": true - }, "version": { "type": "boolean", "description": "Display version and exit.", @@ -474,88 +423,54 @@ "fa_icon": "fas fa-palette", "hidden": true }, - "monochromeLogs": { - "type": "boolean", - "fa_icon": "fas fa-palette", - "description": "Do not use coloured log outputs.", - "hidden": true - }, "hook_url": { "type": "string", "description": "Incoming hook URL for messaging service", "fa_icon": "fas fa-people-group", "hidden": true - }, - "validate_params": { - "type": "boolean", - "description": "Boolean whether to validate parameters against the schema at runtime", - "default": true, - "fa_icon": "fas fa-check-square", - "hidden": true - }, - "validationShowHiddenParams": { - "type": "boolean", - "fa_icon": "far fa-eye-slash", - "description": "Show all params when using `--help`", - "hidden": true - }, - "validationFailUnrecognisedParams": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters fails when an unrecognised parameter is found.", - "hidden": true - }, - "validationLenientMode": { - "type": "boolean", - "fa_icon": "far fa-check-circle", - "description": "Validation of parameters in lenient more.", - "hidden": true } } } }, "allOf": [ { - "$ref": "#/definitions/input_output_options" - }, - { - "$ref": "#/definitions/validation_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/general_stats_options" + "$ref": "#/$defs/validation_options" }, { - "$ref": "#/definitions/ncbi_fcs_options" + "$ref": "#/$defs/general_stats_options" }, { - "$ref": "#/definitions/busco_options" + "$ref": "#/$defs/ncbi_fcs_options" }, { - "$ref": "#/definitions/tidk_options" + "$ref": "#/$defs/busco_options" }, { - "$ref": "#/definitions/lai_options" + "$ref": "#/$defs/tidk_options" }, { - "$ref": "#/definitions/kraken2_options" + "$ref": "#/$defs/lai_options" }, { - "$ref": "#/definitions/hic_options" + "$ref": "#/$defs/kraken2_options" }, { - "$ref": "#/definitions/synteny_options" + "$ref": "#/$defs/hic_options" }, { - "$ref": "#/definitions/merqury_options" + "$ref": "#/$defs/synteny_options" }, { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/$defs/merqury_options" }, { - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/$defs/institutional_config_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/pfr_assemblyqc b/pfr_assemblyqc index 8f40d31d..62c34a3f 100644 --- a/pfr_assemblyqc +++ b/pfr_assemblyqc @@ -41,7 +41,8 @@ if [ $full_test_flag -eq 1 ]; then --ncbi_fcs_gx_db_path "/workspace/ComparativeDataSources/NCBI/FCS/GX/r2023-01-24" \ --kraken2_skip false \ --kraken2_db_path "/workspace/ComparativeDataSources/kraken2db/k2_pluspfp_20230314" \ - -resume + -resume \ + --outdir results else nextflow \ main.nf \ diff --git a/subworkflows/local/utils_nfcore_assemblyqc_pipeline/main.nf b/subworkflows/local/utils_nfcore_assemblyqc_pipeline/main.nf index a8b381e2..c7634358 100644 --- a/subworkflows/local/utils_nfcore_assemblyqc_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_assemblyqc_pipeline/main.nf @@ -10,17 +10,14 @@ import groovy.json.JsonOutput ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' -include { paramsSummaryMap } from 'plugin/nf-validation' -include { fromSamplesheet } from 'plugin/nf-validation' -include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' -include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' -include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' include { imNotification } from '../../nf-core/utils_nfcore_pipeline' include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' -include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' /* ======================================================================================== @@ -32,8 +29,6 @@ workflow PIPELINE_INITIALISATION { take: version // boolean: Display version and exit - help // boolean: Display help text - validate_params // boolean: Boolean whether to validate parameters against the schema at runtime monochrome_logs // boolean: Do not use coloured log outputs nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved @@ -54,21 +49,17 @@ workflow PIPELINE_INITIALISATION { workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 ) + // // Validate parameters and generate parameter summary to stdout // - pre_help_text = nfCoreLogo(monochrome_logs) - post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) - def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input assemblysheet.csv --outdir " - UTILS_NFVALIDATION_PLUGIN ( - help, - workflow_command, - pre_help_text, - post_help_text, - validate_params, - "nextflow_schema.json" + UTILS_NFSCHEMA_PLUGIN ( + workflow, + true, // validate params + null // schema path: nextflow_schema ) + // // Check config provided to the pipeline // @@ -84,7 +75,7 @@ workflow PIPELINE_INITIALISATION { // Initialise input channels // - ch_input = Channel.fromSamplesheet('input') + ch_input = Channel.fromList (samplesheetToList(input, "assets/schema_input.json")) // Function: validateInputTags ch_input_validated = ch_input @@ -109,7 +100,7 @@ workflow PIPELINE_INITIALISATION { ch_xref_assembly = params.synteny_skip || ! params.synteny_xref_assemblies ? Channel.empty() - : Channel.fromSamplesheet('synteny_xref_assemblies') + : Channel.fromList(samplesheetToList(params.synteny_xref_assemblies, "assets/schema_xref_assemblies.json")) ch_xref_assembly_validated = ch_xref_assembly | map { row -> row[0] } @@ -200,12 +191,13 @@ workflow PIPELINE_COMPLETION { email // string: email address email_on_fail // string: email address sent on pipeline failure plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published monochrome_logs // boolean: Disable ANSI colour codes in log output hook_url // string: hook URL for notifications - main: + main: summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") // @@ -213,11 +205,18 @@ workflow PIPELINE_COMPLETION { // workflow.onComplete { if (email || email_on_fail) { - completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs) + completionEmail( + summary_params, + email, + email_on_fail, + plaintext_email, + outdir, + monochrome_logs, + [] + ) } completionSummary(monochrome_logs) - if (hook_url) { imNotification(summary_params, hook_url) } diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 00000000..4994303e --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,46 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + if(parameters_schema) { + log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) + } else { + log.info paramsSummaryLog(input_workflow) + } + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + if(parameters_schema) { + validateParameters(parameters_schema:parameters_schema) + } else { + validateParameters() + } + } + + emit: + dummy_emit = true +} + diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 00000000..f7d9f028 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test new file mode 100644 index 00000000..842dc432 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -0,0 +1,117 @@ +nextflow_workflow { + + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" + script "../main.nf" + workflow "UTILS_NFSCHEMA_PLUGIN" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" + + config "./nextflow.config" + + test("Should run nothing") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params - custom schema") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 00000000..0907ac58 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json similarity index 95% rename from subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json rename to subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json index 7626c1c9..331e0d2f 100644 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", "title": ". pipeline parameters", "description": "", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -87,10 +87,10 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf deleted file mode 100644 index 2585b65d..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ /dev/null @@ -1,62 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsHelp } from 'plugin/nf-validation' -include { paramsSummaryLog } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow UTILS_NFVALIDATION_PLUGIN { - - take: - print_help // boolean: print help - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log - validate_params // boolean: validate parameters - schema_filename // path: JSON schema file, null to use default value - - main: - - log.debug "Using schema file: ${schema_filename}" - - // Default values for strings - pre_help_text = pre_help_text ?: '' - post_help_text = post_help_text ?: '' - workflow_command = workflow_command ?: '' - - // - // Print help message if needed - // - if (print_help) { - log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters(parameters_schema: schema_filename) - } - - emit: - dummy_emit = true -} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml deleted file mode 100644 index 3d4a6b04..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml +++ /dev/null @@ -1,44 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NFVALIDATION_PLUGIN" -description: Use nf-validation to initiate and validate a pipeline -keywords: - - utility - - pipeline - - initialise - - validation -components: [] -input: - - print_help: - type: boolean - description: | - Print help message and exit - - workflow_command: - type: string - description: | - The command to run the workflow e.g. "nextflow run main.nf" - - pre_help_text: - type: string - description: | - Text to print before the help message - - post_help_text: - type: string - description: | - Text to print after the help message - - validate_params: - type: boolean - description: | - Validate the parameters and error if invalid. - - schema_filename: - type: string - description: | - The filename of the schema to validate against. -output: - - dummy_emit: - type: boolean - description: | - Dummy emit to make nf-core subworkflows lint happy -authors: - - "@adamrtalbot" -maintainers: - - "@adamrtalbot" - - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test deleted file mode 100644 index 5784a33f..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ /dev/null @@ -1,200 +0,0 @@ -nextflow_workflow { - - name "Test Workflow UTILS_NFVALIDATION_PLUGIN" - script "../main.nf" - workflow "UTILS_NFVALIDATION_PLUGIN" - tag "subworkflows" - tag "subworkflows_nfcore" - tag "plugin/nf-validation" - tag "'plugin/nf-validation'" - tag "utils_nfvalidation_plugin" - tag "subworkflows/utils_nfvalidation_plugin" - - test("Should run nothing") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - - workflow { - """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success } - ) - } - } - - test("Should run help") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with command") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with extra text") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = "pre-help-text" - post_help_text = "post-help-text" - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('pre-help-text') } }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } }, - { assert workflow.stdout.any { it.contains('post-help-text') } } - ) - } - } - - test("Should validate params") { - - when { - - params { - monochrome_logs = true - test_data = '' - outdir = 1 - } - workflow { - """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = true - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.failed }, - { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } - ) - } - } -} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml deleted file mode 100644 index 60b1cfff..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows/utils_nfvalidation_plugin: - - subworkflows/nf-core/utils_nfvalidation_plugin/** diff --git a/tests/hicparam/hicparam.nf b/tests/hicparam/hicparam.nf index f3080271..b15416b4 100644 --- a/tests/hicparam/hicparam.nf +++ b/tests/hicparam/hicparam.nf @@ -3,7 +3,7 @@ import groovy.json.JsonSlurper def checkHiCParam(paramValue, schema) { def jsonSlurper = new JsonSlurper() def jsonContent = jsonSlurper.parse ( file ( schema, checkIfExists: true ) ) - def pattern = jsonContent.definitions.hic_options.properties.hic.pattern + def pattern = jsonContent['$defs'].hic_options.properties.hic.pattern def match = paramValue ==~ pattern return match diff --git a/tests/hicparam/params.json b/tests/hicparam/params.json index 2f0c2d51..c99d6c4b 100644 --- a/tests/hicparam/params.json +++ b/tests/hicparam/params.json @@ -2,8 +2,5 @@ "config_profile_name": "Test to verify hic param validation", "config_profile_description": "Test to verify hic param validation", "input": "tests/hicparam/assemblysheet.csv", - "hic": "tests/hicparam/hic/Dummy_hic_{1,2}.merged.fq.gz", - "max_cpus": 2, - "max_memory": "6.GB", - "max_time": "6.h" + "hic": "tests/hicparam/hic/Dummy_hic_{1,2}.merged.fq.gz" } diff --git a/tests/invalid/params.json b/tests/invalid/params.json index dd017c16..eb2dab01 100644 --- a/tests/invalid/params.json +++ b/tests/invalid/params.json @@ -1,8 +1,5 @@ { "config_profile_name": "Invalid profile", "config_profile_description": "Profile to test invalid files", - "input": "tests/invalid/assemblysheet.csv", - "max_cpus": 2, - "max_memory": "6.GB", - "max_time": "6.h" + "input": "tests/invalid/assemblysheet.csv" } diff --git a/tests/merqury/mixed2x/params.json b/tests/merqury/mixed2x/params.json index 6ce89d4b..aa96496d 100644 --- a/tests/merqury/mixed2x/params.json +++ b/tests/merqury/mixed2x/params.json @@ -3,8 +3,5 @@ "config_profile_description": "Merqury test for a mixed diploid assembly contained in a single fasta", "input": "tests/merqury/mixed2x/assemblysheet.csv", "merqury_skip": false, - "merqury_kmer_length": 21, - "max_cpus": 8, - "max_memory": "32.GB", - "max_time": "6.h" + "merqury_kmer_length": 21 } diff --git a/tests/merqury/phased2x.mp/params.json b/tests/merqury/phased2x.mp/params.json index 60f157ca..0b4479f0 100644 --- a/tests/merqury/phased2x.mp/params.json +++ b/tests/merqury/phased2x.mp/params.json @@ -3,8 +3,5 @@ "config_profile_description": "Merqury test for a phased diploid assembly contained in separate fasta with parental reads", "input": "tests/merqury/phased2x.mp/assemblysheet.csv", "merqury_skip": false, - "merqury_kmer_length": 21, - "max_cpus": 8, - "max_memory": "32.GB", - "max_time": "6.h" + "merqury_kmer_length": 21 } diff --git a/tests/merqury/phased2x/params.json b/tests/merqury/phased2x/params.json index 0b363fab..e37db9ab 100644 --- a/tests/merqury/phased2x/params.json +++ b/tests/merqury/phased2x/params.json @@ -3,8 +3,5 @@ "config_profile_description": "Merqury test for a phased diploid assembly contained in separate fasta", "input": "tests/merqury/phased2x/assemblysheet.csv", "merqury_skip": false, - "merqury_kmer_length": 21, - "max_cpus": 8, - "max_memory": "32.GB", - "max_time": "6.h" + "merqury_kmer_length": 21 } diff --git a/tests/minimal/params.json b/tests/minimal/params.json index c5683622..0b8c5117 100644 --- a/tests/minimal/params.json +++ b/tests/minimal/params.json @@ -1,8 +1,5 @@ { "config_profile_name": "Test profile", "config_profile_description": "Minimal test dataset to check pipeline function", - "input": "https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/assets/assemblysheetv2.csv", - "max_cpus": 2, - "max_memory": "6.GB", - "max_time": "6.h" + "input": "https://raw.githubusercontent.com/plant-food-research-open/assemblyqc/dev/assets/assemblysheetv2.csv" } diff --git a/tests/noltr/params.json b/tests/noltr/params.json index a46bd34d..e52470db 100644 --- a/tests/noltr/params.json +++ b/tests/noltr/params.json @@ -2,8 +2,5 @@ "config_profile_name": "No LTRs assembly profile", "config_profile_description": "Profile to test an assembly without LTRs", "input": "tests/noltr/assemblysheet.csv", - "lai_skip": false, - "max_cpus": 2, - "max_memory": "6.GB", - "max_time": "6.h" + "lai_skip": false } diff --git a/tests/stub/params.json b/tests/stub/params.json index 573986a7..85bdf0ce 100644 --- a/tests/stub/params.json +++ b/tests/stub/params.json @@ -23,8 +23,5 @@ "synteny_mummer_skip": false, "synteny_plotsr_skip": false, "synteny_xref_assemblies": "assets/xrefsheet.csv", - "merqury_skip": false, - "max_cpus": 2, - "max_memory": "6.GB", - "max_time": "6.h" + "merqury_skip": false }