From b58e6d68580ce2a30fb9133d915705d0968b1b90 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 16:35:43 +0100 Subject: [PATCH 01/14] Adding sanger configs --- .github/workflows/sanger_test.yml | 29 +++++++++++++++++ .github/workflows/sanger_test_full.yml | 43 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 .github/workflows/sanger_test.yml create mode 100644 .github/workflows/sanger_test_full.yml diff --git a/.github/workflows/sanger_test.yml b/.github/workflows/sanger_test.yml new file mode 100644 index 00000000..adb5ce90 --- /dev/null +++ b/.github/workflows/sanger_test.yml @@ -0,0 +1,29 @@ +name: sanger-tol LSF tests + +on: + workflow_dispatch: +jobs: + run-tower: + name: Run LSF tests + runs-on: ubuntu-latest + steps: + - name: Launch workflow via tower + uses: seqeralabs/action-tower-launch@v2 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} + workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ github.sha }} + parameters: | + { + "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ github.sha }}", + } + profiles: farm_test,sanger,singularity,cleanup + + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: | + tower_action_*.log + tower_action_*.json diff --git a/.github/workflows/sanger_test_full.yml b/.github/workflows/sanger_test_full.yml new file mode 100644 index 00000000..0f606568 --- /dev/null +++ b/.github/workflows/sanger_test_full.yml @@ -0,0 +1,43 @@ +name: sanger-tol LSF full size tests + +on: + push: + branches: + - main + - dev + workflow_dispatch: +jobs: + run-tower: + name: Run LSF full size tests + runs-on: ubuntu-latest + steps: + - name: Sets env vars for push + run: | + echo "REVISION=${{ github.sha }}" >> $GITHUB_ENV + if: github.event_name == 'push' + + - name: Sets env vars for workflow_dispatch + run: | + echo "REVISION=${{ github.sha }}" >> $GITHUB_ENV + if: github.event_name == 'workflow_dispatch' + + - name: Launch workflow via tower + uses: seqeralabs/action-tower-launch@v2 + with: + workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} + access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} + compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ env.REVISION }} + workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ env.REVISION }} + parameters: | + { + "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}", + } + profiles: farm_test,sanger,singularity,cleanup + + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: | + tower_action_*.log + tower_action_*.json From 44518368b664b52ac4e96b1da385d6cbc00e78de Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 16:35:56 +0100 Subject: [PATCH 02/14] Updates for new cli --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8bc073a8..08a85663 100755 --- a/README.md +++ b/README.md @@ -39,7 +39,11 @@ Currently, it is advised to run the pipeline with docker or singularity as a sma Now, you can run the pipeline using: ```bash -nextflow run main.nf -profile singularity --input treeval.yaml -entry {FULL|RAPID} --outdir {OUTDIR} +# For the FULL pipeline +nextflow run main.nf -profile singularity --input treeval.yaml --outdir {OUTDIR} + +# For the RAPID subset +nextflow run main.nf -profile singularity --input treeval.yaml -entry RAPID --outdir {OUTDIR} ``` An example treeval.yaml can be found [here](assets/local_testing/nxOscDF5033.yaml). From febf2271d4c49290dc5191c4c04695615ea4a235 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 16:36:13 +0100 Subject: [PATCH 03/14] Updates for new cli and sanger testing --- nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nextflow.config b/nextflow.config index 17035f39..fcd4e6fc 100755 --- a/nextflow.config +++ b/nextflow.config @@ -60,6 +60,7 @@ try { profiles { + cleanup { cleanup = true } debug { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' From 27ea3bc43f07610531baf891b07195bd73992283 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 16:37:20 +0100 Subject: [PATCH 04/14] Updates to testing --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89b33330..9d7b7c9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,4 +48,4 @@ jobs: - name: Run FULL pipeline with test data # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -entry FULL -profile github_test,docker --outdir ./results-full + nextflow run ${GITHUB_WORKSPACE} -profile github_test,docker --outdir ./results-full From 6976bf2439398fdcb1252a1309803b340a01d4d9 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 16:37:39 +0100 Subject: [PATCH 05/14] Updating the paths in files --- assets/local_testing/nxOscDF5033.yaml | 7 +++---- assets/local_testing/nxOscSUBSET.yaml | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/assets/local_testing/nxOscDF5033.yaml b/assets/local_testing/nxOscDF5033.yaml index 8c2547c1..6754146d 100755 --- a/assets/local_testing/nxOscDF5033.yaml +++ b/assets/local_testing/nxOscDF5033.yaml @@ -5,11 +5,10 @@ assembly: classT: nematode asmVersion: 1 gevalType: DTOL -reference_file: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta -#/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta +reference_file: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta assem_reads: - pacbio: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/ - hic: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/ + pacbio: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/ + hic: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/ supplementary: path alignment: data_dir: /lustre/scratch123/tol/resources/treeval/gene_alignment_data/ diff --git a/assets/local_testing/nxOscSUBSET.yaml b/assets/local_testing/nxOscSUBSET.yaml index fcc726b1..45e3a554 100755 --- a/assets/local_testing/nxOscSUBSET.yaml +++ b/assets/local_testing/nxOscSUBSET.yaml @@ -5,11 +5,11 @@ assembly: classT: nematode asmVersion: 1 gevalType: DTOL -reference_file: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta +reference_file: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta #/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta assem_reads: - pacbio: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/genomic_data/pacbio/ - hic: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/ + pacbio: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_SUBSET/genomic_data/pacbio/ + hic: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/ supplementary: path alignment: data_dir: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/gene_set/ From 1f41c53174ca5c17cb78d601bbfd8c10e7ba0217 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 17:14:59 +0100 Subject: [PATCH 06/14] removing the workflow name --- main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.nf b/main.nf index 0d25a633..9a7b3fba 100755 --- a/main.nf +++ b/main.nf @@ -48,7 +48,7 @@ workflow SANGERTOL_TREEVAL_RAPID { // // WORKFLOWS: Execute named workflow for the pipeline // -workflow FULL { +workflow { SANGERTOL_TREEVAL () } From b3d52a7f86c43e51b824f713fc108e65b3c4b5fb Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 17:19:46 +0100 Subject: [PATCH 07/14] Renaming test.configs --- conf/test.config | 12 +++++++----- conf/test_full.config | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) create mode 100755 conf/test_full.config diff --git a/conf/test.config b/conf/test.config index c3a6bb5e..9966e48d 100755 --- a/conf/test.config +++ b/conf/test.config @@ -14,13 +14,15 @@ */ params { - config_profile_name = 'Test profile' - config_profile_description = 'Minimal test dataset to check pipeline function' + config_profile_name = 'GitHub FULL test' + config_profile_description = 'FULL Test Data for GitHub Actions test' // Limit resources so that this can run on GitHub Actions - max_cpus = 6 - max_memory = '12.GB' + max_cpus = 2 + max_memory = '6.GB' max_time = '6.h' - input = 'assets/local_testing/nxOscSUBSET.yaml' + // Input data + input = "${projectDir}/assets/github_testing/TreeValTinyTest.yaml" + outdir = "TinyTest" } diff --git a/conf/test_full.config b/conf/test_full.config new file mode 100755 index 00000000..68b15e3a --- /dev/null +++ b/conf/test_full.config @@ -0,0 +1,24 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Nextflow config file for running full-size tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Defines input files and everything required to run a full size pipeline test. + + Use as follows: + nextflow run sanger-tol/treeval -profile farm_full,singularity,sanger + + On LSF / tol farm: + bsub -Is -tty -e error -o out -n 2 -q oversubscribed -M4000 -R'select[mem>4000] rusage[mem=4000] span[hosts=1]' 'nextflow run main.nf -profile test_full,singularity,sanger' + +---------------------------------------------------------------------------------------- +*/ + +cleanup = true + +params { + config_profile_name = "FULL local test profile" + config_profile_description = "FULL test dataset to check pipeline function, using a current full local dataset" + + input = "${projectDir}/assets/local_testing/nxOscDF5033.yaml" + output = nxOscDF5033 +} From cad8ea637321898da99502a0f06110460c995d61 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Tue, 19 Sep 2023 17:23:36 +0100 Subject: [PATCH 08/14] Updating --- .github/workflows/sanger_test.yml | 2 +- .github/workflows/sanger_test_full.yml | 2 +- conf/farm_test.config | 24 ------------------------ nextflow.config | 3 ++- 4 files changed, 4 insertions(+), 27 deletions(-) delete mode 100755 conf/farm_test.config diff --git a/.github/workflows/sanger_test.yml b/.github/workflows/sanger_test.yml index adb5ce90..e69af1ef 100644 --- a/.github/workflows/sanger_test.yml +++ b/.github/workflows/sanger_test.yml @@ -19,7 +19,7 @@ jobs: { "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ github.sha }}", } - profiles: farm_test,sanger,singularity,cleanup + profiles: test,sanger,singularity,cleanup - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/sanger_test_full.yml b/.github/workflows/sanger_test_full.yml index 0f606568..e028c6b6 100644 --- a/.github/workflows/sanger_test_full.yml +++ b/.github/workflows/sanger_test_full.yml @@ -33,7 +33,7 @@ jobs: { "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}", } - profiles: farm_test,sanger,singularity,cleanup + profiles: test_full,sanger,singularity,cleanup - uses: actions/upload-artifact@v3 with: diff --git a/conf/farm_test.config b/conf/farm_test.config deleted file mode 100755 index 68b15e3a..00000000 --- a/conf/farm_test.config +++ /dev/null @@ -1,24 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running full-size tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a full size pipeline test. - - Use as follows: - nextflow run sanger-tol/treeval -profile farm_full,singularity,sanger - - On LSF / tol farm: - bsub -Is -tty -e error -o out -n 2 -q oversubscribed -M4000 -R'select[mem>4000] rusage[mem=4000] span[hosts=1]' 'nextflow run main.nf -profile test_full,singularity,sanger' - ----------------------------------------------------------------------------------------- -*/ - -cleanup = true - -params { - config_profile_name = "FULL local test profile" - config_profile_description = "FULL test dataset to check pipeline function, using a current full local dataset" - - input = "${projectDir}/assets/local_testing/nxOscDF5033.yaml" - output = nxOscDF5033 -} diff --git a/nextflow.config b/nextflow.config index fcd4e6fc..90516ac0 100755 --- a/nextflow.config +++ b/nextflow.config @@ -152,7 +152,8 @@ profiles { executor.memory = 60.GB } - farm_test { includeConfig 'conf/farm_test.config' } + test { includeConfig 'conf/test.config' } + test_full { includeConfig 'conf/farm_test.config' } github_test { includeConfig 'conf/github_test.config' } local_github_test { includeConfig 'conf/local_github_test.config' } } From d7a448f467d445bca9de65ac373f70d6bbe20879 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Wed, 20 Sep 2023 10:13:10 +0100 Subject: [PATCH 09/14] Updates for sanger testing --- .github/workflows/ci.yml | 4 +-- .github/workflows/sanger_test.yml | 29 ----------------- .../github_testing/TreeValTinyTest-Local.yaml | 31 ------------------- conf/github_test.config | 25 --------------- conf/local_github_test.config | 25 --------------- nextflow.config | 2 -- 6 files changed, 2 insertions(+), 114 deletions(-) delete mode 100644 .github/workflows/sanger_test.yml delete mode 100755 assets/github_testing/TreeValTinyTest-Local.yaml delete mode 100755 conf/github_test.config delete mode 100755 conf/local_github_test.config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d7b7c9b..c082bc5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,9 +43,9 @@ jobs: - name: Run RAPID pipeline with test data # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -entry RAPID -profile github_test,docker --outdir ./results-rapid + nextflow run ${GITHUB_WORKSPACE} -entry RAPID -profile test,docker --outdir ./results-rapid - name: Run FULL pipeline with test data # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile github_test,docker --outdir ./results-full + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results-full diff --git a/.github/workflows/sanger_test.yml b/.github/workflows/sanger_test.yml deleted file mode 100644 index e69af1ef..00000000 --- a/.github/workflows/sanger_test.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: sanger-tol LSF tests - -on: - workflow_dispatch: -jobs: - run-tower: - name: Run LSF tests - runs-on: ubuntu-latest - steps: - - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2 - with: - workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} - access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} - compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} - revision: ${{ github.sha }} - workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ github.sha }} - parameters: | - { - "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ github.sha }}", - } - profiles: test,sanger,singularity,cleanup - - - uses: actions/upload-artifact@v3 - with: - name: Tower debug log file - path: | - tower_action_*.log - tower_action_*.json diff --git a/assets/github_testing/TreeValTinyTest-Local.yaml b/assets/github_testing/TreeValTinyTest-Local.yaml deleted file mode 100755 index 532d0fd6..00000000 --- a/assets/github_testing/TreeValTinyTest-Local.yaml +++ /dev/null @@ -1,31 +0,0 @@ -assembly: - level: scaffold - sample_id: grTriPseu1 - latin_name: to_provide_taxonomic_rank - classT: fungi - asmVersion: 1 - dbVersion: "1" - gevalType: DTOL -reference_file: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/assembly/draft/grTriPseu1.fa -assem_reads: - pacbio: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/genomic_data/pacbio/ - hic: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/genomic_data/hic-arima/ - supplementary: path -alignment: - data_dir: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/gene_alignment_data/ - common_name: "" # For future implementation (adding bee, wasp, ant etc) - geneset: "LaetiporusSulphureus.gfLaeSulp1" - #Path should end up looking like "{data_dir}{classT}/{common_name}/csv_data/{geneset}-data.csv" -self_comp: - motif_len: 0 - mummer_chunk: 10 -synteny: - synteny_genome_path: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/synteny/ -outdir: "NEEDS TESTING" -intron: - size: "50k" -telomere: - teloseq: TTAGGG -busco: - lineages_path: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/busco/subset/ - lineage: fungi_odb10 diff --git a/conf/github_test.config b/conf/github_test.config deleted file mode 100755 index a5acbb0d..00000000 --- a/conf/github_test.config +++ /dev/null @@ -1,25 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run sanger-tol/treeval -profile s3_test, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'GitHub FULL test' - config_profile_description = 'FULL Test Data for GitHub Actions test' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = "${projectDir}/assets/github_testing/TreeValTinyTest.yaml" - outdir = "TinyTest" -} diff --git a/conf/local_github_test.config b/conf/local_github_test.config deleted file mode 100755 index 8544b962..00000000 --- a/conf/local_github_test.config +++ /dev/null @@ -1,25 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Nextflow config file for running minimal tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run sanger-tol/treeval -profile s3_test, --outdir - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'GitHub FULL test' - config_profile_description = 'FULL Test Data for GitHub Actions test' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = "${projectDir}/assets/github_testing/TreeValTinyTest-Local.yaml" - outdir = "TinyTest" -} diff --git a/nextflow.config b/nextflow.config index 90516ac0..b13595f6 100755 --- a/nextflow.config +++ b/nextflow.config @@ -154,8 +154,6 @@ profiles { test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/farm_test.config' } - github_test { includeConfig 'conf/github_test.config' } - local_github_test { includeConfig 'conf/local_github_test.config' } } From eba5e183ce3333907b59910649bc8d394984cd59 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Wed, 20 Sep 2023 15:39:27 +0100 Subject: [PATCH 10/14] Updates to testing --- assets/local_testing/nxOscDF5033.yaml | 6 +++--- assets/local_testing/nxOscSUBSET.yaml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/assets/local_testing/nxOscDF5033.yaml b/assets/local_testing/nxOscDF5033.yaml index 6754146d..86ac7182 100755 --- a/assets/local_testing/nxOscDF5033.yaml +++ b/assets/local_testing/nxOscDF5033.yaml @@ -5,10 +5,10 @@ assembly: classT: nematode asmVersion: 1 gevalType: DTOL -reference_file: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta +reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta assem_reads: - pacbio: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/ - hic: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/ + pacbio: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/ + hic: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/ supplementary: path alignment: data_dir: /lustre/scratch123/tol/resources/treeval/gene_alignment_data/ diff --git a/assets/local_testing/nxOscSUBSET.yaml b/assets/local_testing/nxOscSUBSET.yaml index 45e3a554..6284a4ce 100755 --- a/assets/local_testing/nxOscSUBSET.yaml +++ b/assets/local_testing/nxOscSUBSET.yaml @@ -5,14 +5,14 @@ assembly: classT: nematode asmVersion: 1 gevalType: DTOL -reference_file: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta +reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta #/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta assem_reads: - pacbio: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_SUBSET/genomic_data/pacbio/ - hic: /lustre/scratch123/tol/resources/nextflow/test-data/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/ + pacbio: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/genomic_data/pacbio/ + hic: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/ supplementary: path alignment: - data_dir: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/gene_set/ + data_dir: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/gene_set/ common_name: "" # For future implementation (adding bee, wasp, ant etc) geneset: "Gae_host.Gae" #Path should end up looking like "{data_dir}{classT}/{common_name}/csv_data/{geneset}-data.csv" From 82f2858396fbf9ed0990c9fccd505c57f3b12030 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Wed, 20 Sep 2023 15:39:44 +0100 Subject: [PATCH 11/14] Updates to testing --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c082bc5d..3ddc80d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: - name: Download test data # Download A fungal test data set that is full enough to show some real output. run: | - curl https://dp24.cog.sanger.ac.uk/TreeValTinyData.tar.gz | tar xzf - + curl https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData.tar.gz | tar xzf - - name: Run RAPID pipeline with test data # Remember that you can parallelise this by using strategy.matrix From 8e8f64b246e5abe862398f2dbe26a6fc29feaa83 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Wed, 20 Sep 2023 15:39:59 +0100 Subject: [PATCH 12/14] Updating README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08a85663..436826b3 100755 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ sanger-tol/treeval has been written by Damon-Lee Pointon (@DLBPointon), Yumi Sim We thank the following people for their extensive assistance in the development of this pipeline:
    -
  • @gq1 - For building the infrastructure around TreeVal
    • +
  • @gq1 - For building the infrastructure around TreeVal and helping with code review
  • @ksenia-krasheninnikova - For help with C code implementation and YAML parsing
  • @mcshane - For guidance on algorithms
  • @muffato - For code reviews and code support
    • From 5e11acaf830974e5db8276084cd558de95b0a3a7 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Thu, 21 Sep 2023 10:50:42 +0100 Subject: [PATCH 13/14] Correcting the test file --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index b13595f6..924fcff7 100755 --- a/nextflow.config +++ b/nextflow.config @@ -153,7 +153,7 @@ profiles { } test { includeConfig 'conf/test.config' } - test_full { includeConfig 'conf/farm_test.config' } + test_full { includeConfig 'conf/test_full.config' } } From 718acec623516822bbac4fd0a404c7e6156d3321 Mon Sep 17 00:00:00 2001 From: DLBPointon Date: Thu, 21 Sep 2023 10:56:31 +0100 Subject: [PATCH 14/14] Correcting the test file --- conf/test_full.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/test_full.config b/conf/test_full.config index 68b15e3a..deb9aac9 100755 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -20,5 +20,5 @@ params { config_profile_description = "FULL test dataset to check pipeline function, using a current full local dataset" input = "${projectDir}/assets/local_testing/nxOscDF5033.yaml" - output = nxOscDF5033 + outdir = "SmallTest" }