diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 89b33330..3ddc80d9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -38,14 +38,14 @@ jobs:
- name: Download test data
# Download A fungal test data set that is full enough to show some real output.
run: |
- curl https://dp24.cog.sanger.ac.uk/TreeValTinyData.tar.gz | tar xzf -
+ curl https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData.tar.gz | tar xzf -
- name: Run RAPID pipeline with test data
# Remember that you can parallelise this by using strategy.matrix
run: |
- nextflow run ${GITHUB_WORKSPACE} -entry RAPID -profile github_test,docker --outdir ./results-rapid
+ nextflow run ${GITHUB_WORKSPACE} -entry RAPID -profile test,docker --outdir ./results-rapid
- name: Run FULL pipeline with test data
# Remember that you can parallelise this by using strategy.matrix
run: |
- nextflow run ${GITHUB_WORKSPACE} -entry FULL -profile github_test,docker --outdir ./results-full
+ nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results-full
diff --git a/.github/workflows/sanger_test_full.yml b/.github/workflows/sanger_test_full.yml
new file mode 100644
index 00000000..e028c6b6
--- /dev/null
+++ b/.github/workflows/sanger_test_full.yml
@@ -0,0 +1,43 @@
+name: sanger-tol LSF full size tests
+
+on:
+ push:
+ branches:
+ - main
+ - dev
+ workflow_dispatch:
+jobs:
+ run-tower:
+ name: Run LSF full size tests
+ runs-on: ubuntu-latest
+ steps:
+ - name: Sets env vars for push
+ run: |
+ echo "REVISION=${{ github.sha }}" >> $GITHUB_ENV
+ if: github.event_name == 'push'
+
+ - name: Sets env vars for workflow_dispatch
+ run: |
+ echo "REVISION=${{ github.sha }}" >> $GITHUB_ENV
+ if: github.event_name == 'workflow_dispatch'
+
+ - name: Launch workflow via tower
+ uses: seqeralabs/action-tower-launch@v2
+ with:
+ workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+ access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+ compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+ revision: ${{ env.REVISION }}
+ workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ env.REVISION }}
+ parameters: |
+ {
+ "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}",
+ }
+ profiles: test_full,sanger,singularity,cleanup
+
+ - uses: actions/upload-artifact@v3
+ with:
+ name: Tower debug log file
+ path: |
+ tower_action_*.log
+ tower_action_*.json
diff --git a/README.md b/README.md
index 8bc073a8..436826b3 100755
--- a/README.md
+++ b/README.md
@@ -39,7 +39,11 @@ Currently, it is advised to run the pipeline with docker or singularity as a sma
Now, you can run the pipeline using:
```bash
-nextflow run main.nf -profile singularity --input treeval.yaml -entry {FULL|RAPID} --outdir {OUTDIR}
+# For the FULL pipeline
+nextflow run main.nf -profile singularity --input treeval.yaml --outdir {OUTDIR}
+
+# For the RAPID subset
+nextflow run main.nf -profile singularity --input treeval.yaml -entry RAPID --outdir {OUTDIR}
```
An example treeval.yaml can be found [here](assets/local_testing/nxOscDF5033.yaml).
@@ -58,7 +62,7 @@ sanger-tol/treeval has been written by Damon-Lee Pointon (@DLBPointon), Yumi Sim
We thank the following people for their extensive assistance in the development of this pipeline:
- - @gq1 - For building the infrastructure around TreeVal
+ - @gq1 - For building the infrastructure around TreeVal and helping with code review
- @ksenia-krasheninnikova - For help with C code implementation and YAML parsing
- @mcshane - For guidance on algorithms
- @muffato - For code reviews and code support
diff --git a/assets/github_testing/TreeValTinyTest-Local.yaml b/assets/github_testing/TreeValTinyTest-Local.yaml
deleted file mode 100755
index 532d0fd6..00000000
--- a/assets/github_testing/TreeValTinyTest-Local.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-assembly:
- level: scaffold
- sample_id: grTriPseu1
- latin_name: to_provide_taxonomic_rank
- classT: fungi
- asmVersion: 1
- dbVersion: "1"
- gevalType: DTOL
-reference_file: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/assembly/draft/grTriPseu1.fa
-assem_reads:
- pacbio: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/genomic_data/pacbio/
- hic: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/genomic_data/hic-arima/
- supplementary: path
-alignment:
- data_dir: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/gene_alignment_data/
- common_name: "" # For future implementation (adding bee, wasp, ant etc)
- geneset: "LaetiporusSulphureus.gfLaeSulp1"
- #Path should end up looking like "{data_dir}{classT}/{common_name}/csv_data/{geneset}-data.csv"
-self_comp:
- motif_len: 0
- mummer_chunk: 10
-synteny:
- synteny_genome_path: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/synteny/
-outdir: "NEEDS TESTING"
-intron:
- size: "50k"
-telomere:
- teloseq: TTAGGG
-busco:
- lineages_path: /nfs/treeoflife-01/teams/tola/users/dp24/treeval/TreeValTinyData/busco/subset/
- lineage: fungi_odb10
diff --git a/assets/local_testing/nxOscDF5033.yaml b/assets/local_testing/nxOscDF5033.yaml
index 8c2547c1..86ac7182 100755
--- a/assets/local_testing/nxOscDF5033.yaml
+++ b/assets/local_testing/nxOscDF5033.yaml
@@ -5,11 +5,10 @@ assembly:
classT: nematode
asmVersion: 1
gevalType: DTOL
-reference_file: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
-#/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
+reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
assem_reads:
- pacbio: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/
- hic: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/
+ pacbio: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/
+ hic: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/
supplementary: path
alignment:
data_dir: /lustre/scratch123/tol/resources/treeval/gene_alignment_data/
diff --git a/assets/local_testing/nxOscSUBSET.yaml b/assets/local_testing/nxOscSUBSET.yaml
index fcc726b1..6284a4ce 100755
--- a/assets/local_testing/nxOscSUBSET.yaml
+++ b/assets/local_testing/nxOscSUBSET.yaml
@@ -5,14 +5,14 @@ assembly:
classT: nematode
asmVersion: 1
gevalType: DTOL
-reference_file: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta
+reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta
#/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
assem_reads:
- pacbio: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/genomic_data/pacbio/
- hic: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/
+ pacbio: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/genomic_data/pacbio/
+ hic: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/
supplementary: path
alignment:
- data_dir: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/gene_set/
+ data_dir: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/gene_set/
common_name: "" # For future implementation (adding bee, wasp, ant etc)
geneset: "Gae_host.Gae"
#Path should end up looking like "{data_dir}{classT}/{common_name}/csv_data/{geneset}-data.csv"
diff --git a/conf/github_test.config b/conf/github_test.config
deleted file mode 100755
index a5acbb0d..00000000
--- a/conf/github_test.config
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Nextflow config file for running minimal tests
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Defines input files and everything required to run a fast and simple pipeline test.
-
- Use as follows:
- nextflow run sanger-tol/treeval -profile s3_test, --outdir
-
-----------------------------------------------------------------------------------------
-*/
-
-params {
- config_profile_name = 'GitHub FULL test'
- config_profile_description = 'FULL Test Data for GitHub Actions test'
-
- // Limit resources so that this can run on GitHub Actions
- max_cpus = 2
- max_memory = '6.GB'
- max_time = '6.h'
-
- // Input data
- input = "${projectDir}/assets/github_testing/TreeValTinyTest.yaml"
- outdir = "TinyTest"
-}
diff --git a/conf/local_github_test.config b/conf/local_github_test.config
deleted file mode 100755
index 8544b962..00000000
--- a/conf/local_github_test.config
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Nextflow config file for running minimal tests
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Defines input files and everything required to run a fast and simple pipeline test.
-
- Use as follows:
- nextflow run sanger-tol/treeval -profile s3_test, --outdir
-
-----------------------------------------------------------------------------------------
-*/
-
-params {
- config_profile_name = 'GitHub FULL test'
- config_profile_description = 'FULL Test Data for GitHub Actions test'
-
- // Limit resources so that this can run on GitHub Actions
- max_cpus = 2
- max_memory = '6.GB'
- max_time = '6.h'
-
- // Input data
- input = "${projectDir}/assets/github_testing/TreeValTinyTest-Local.yaml"
- outdir = "TinyTest"
-}
diff --git a/conf/test.config b/conf/test.config
index c3a6bb5e..9966e48d 100755
--- a/conf/test.config
+++ b/conf/test.config
@@ -14,13 +14,15 @@
*/
params {
- config_profile_name = 'Test profile'
- config_profile_description = 'Minimal test dataset to check pipeline function'
+ config_profile_name = 'GitHub FULL test'
+ config_profile_description = 'FULL Test Data for GitHub Actions test'
// Limit resources so that this can run on GitHub Actions
- max_cpus = 6
- max_memory = '12.GB'
+ max_cpus = 2
+ max_memory = '6.GB'
max_time = '6.h'
- input = 'assets/local_testing/nxOscSUBSET.yaml'
+ // Input data
+ input = "${projectDir}/assets/github_testing/TreeValTinyTest.yaml"
+ outdir = "TinyTest"
}
diff --git a/conf/farm_test.config b/conf/test_full.config
similarity index 95%
rename from conf/farm_test.config
rename to conf/test_full.config
index 68b15e3a..deb9aac9 100755
--- a/conf/farm_test.config
+++ b/conf/test_full.config
@@ -20,5 +20,5 @@ params {
config_profile_description = "FULL test dataset to check pipeline function, using a current full local dataset"
input = "${projectDir}/assets/local_testing/nxOscDF5033.yaml"
- output = nxOscDF5033
+ outdir = "SmallTest"
}
diff --git a/main.nf b/main.nf
index 0d25a633..9a7b3fba 100755
--- a/main.nf
+++ b/main.nf
@@ -48,7 +48,7 @@ workflow SANGERTOL_TREEVAL_RAPID {
//
// WORKFLOWS: Execute named workflow for the pipeline
//
-workflow FULL {
+workflow {
SANGERTOL_TREEVAL ()
}
diff --git a/nextflow.config b/nextflow.config
index 17035f39..924fcff7 100755
--- a/nextflow.config
+++ b/nextflow.config
@@ -60,6 +60,7 @@ try {
profiles {
+ cleanup { cleanup = true }
debug {
dumpHashes = true
process.beforeScript = 'echo $HOSTNAME'
@@ -151,9 +152,8 @@ profiles {
executor.memory = 60.GB
}
- farm_test { includeConfig 'conf/farm_test.config' }
- github_test { includeConfig 'conf/github_test.config' }
- local_github_test { includeConfig 'conf/local_github_test.config' }
+ test { includeConfig 'conf/test.config' }
+ test_full { includeConfig 'conf/test_full.config' }
}