Merge pull request #142 from sanger-tol/tower

Tower

.github/workflows/ci.yml

@@ -38,14 +38,14 @@ jobs:
       - name: Download test data
         # Download A fungal test data set that is full enough to show some real output.
         run: |
-          curl https://dp24.cog.sanger.ac.uk/TreeValTinyData.tar.gz | tar xzf -
+          curl https://tolit.cog.sanger.ac.uk/test-data/resources/treeval/TreeValTinyData.tar.gz | tar xzf -
 
       - name: Run RAPID pipeline with test data
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -entry RAPID -profile github_test,docker --outdir ./results-rapid
+          nextflow run ${GITHUB_WORKSPACE} -entry RAPID -profile test,docker --outdir ./results-rapid
 
       - name: Run FULL pipeline with test data
         # Remember that you can parallelise this by using strategy.matrix
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -entry FULL -profile github_test,docker --outdir ./results-full
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results-full

.github/workflows/sanger_test_full.yml

@@ -0,0 +1,43 @@
+name: sanger-tol LSF full size tests
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+  workflow_dispatch:
+jobs:
+  run-tower:
+    name: Run LSF full size tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Sets env vars for push
+        run: |
+          echo "REVISION=${{ github.sha }}" >> $GITHUB_ENV
+        if: github.event_name == 'push'
+
+      - name: Sets env vars for workflow_dispatch
+        run: |
+          echo "REVISION=${{ github.sha }}" >> $GITHUB_ENV
+        if: github.event_name == 'workflow_dispatch'
+
+      - name: Launch workflow via tower
+        uses: seqeralabs/action-tower-launch@v2
+        with:
+          workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
+          access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
+          compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
+          revision: ${{ env.REVISION }}
+          workdir: ${{ secrets.TOWER_WORKDIR_PARENT }}/work/${{ github.repository }}/work-${{ env.REVISION }}
+          parameters: |
+            {
+              "outdir": "${{ secrets.TOWER_WORKDIR_PARENT }}/results/${{ github.repository }}/results-${{ env.REVISION }}",
+            }
+          profiles: test_full,sanger,singularity,cleanup
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Tower debug log file
+          path: |
+            tower_action_*.log
+            tower_action_*.json

README.md

@@ -39,7 +39,11 @@ Currently, it is advised to run the pipeline with docker or singularity as a sma
 Now, you can run the pipeline using:
 
 ```bash
-nextflow run main.nf -profile singularity --input treeval.yaml -entry {FULL|RAPID} --outdir {OUTDIR}
+# For the FULL pipeline
+nextflow run main.nf -profile singularity --input treeval.yaml --outdir {OUTDIR}
+
+# For the RAPID subset
+nextflow run main.nf -profile singularity --input treeval.yaml -entry RAPID --outdir {OUTDIR}
 ```
 
 An example treeval.yaml can be found [here](assets/local_testing/nxOscDF5033.yaml).
@@ -58,7 +62,7 @@ sanger-tol/treeval has been written by Damon-Lee Pointon (@DLBPointon), Yumi Sim
 We thank the following people for their extensive assistance in the development of this pipeline:
 
 <ul>
-  <li>@gq1 - For building the infrastructure around TreeVal</li>
+  <li>@gq1 - For building the infrastructure around TreeVal and helping with code review</li>
   <li>@ksenia-krasheninnikova - For help with C code implementation and YAML parsing</li>
   <li>@mcshane - For guidance on algorithms </li>
   <li>@muffato - For code reviews and code support</li>

assets/local_testing/nxOscDF5033.yaml

@@ -5,11 +5,10 @@ assembly:
   classT: nematode
   asmVersion: 1
   gevalType: DTOL
-reference_file: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
-#/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
+reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
 assem_reads:
-  pacbio: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/
-  hic: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/
+  pacbio: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/pacbio/fasta/
+  hic: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/full/
   supplementary: path
 alignment:
   data_dir: /lustre/scratch123/tol/resources/treeval/gene_alignment_data/

assets/local_testing/nxOscSUBSET.yaml

@@ -5,14 +5,14 @@ assembly:
   classT: nematode
   asmVersion: 1
   gevalType: DTOL
-reference_file: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta
+reference_file: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/assembly/draft/SUBSET_genome/Oscheius_SUBSET.fasta
 #/lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/assembly/draft/DF5033.hifiasm.noTelos.20211120/DF5033.noTelos.hifiasm.purged.noCont.noMito.fasta
 assem_reads:
-  pacbio: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/genomic_data/pacbio/
-  hic: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/
+  pacbio: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/genomic_data/pacbio/
+  hic: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_DF5033/genomic_data/nxOscSpes1/hic-arima2/subset/
   supplementary: path
 alignment:
-  data_dir: /lustre/scratch123/tol/resources/treeval/nextflow_test_data/Oscheius_SUBSET/gene_set/
+  data_dir: /lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValSmallData/Oscheius_SUBSET/gene_set/
   common_name: "" # For future implementation (adding bee, wasp, ant etc)
   geneset: "Gae_host.Gae"
   #Path should end up looking like "{data_dir}{classT}/{common_name}/csv_data/{geneset}-data.csv"

conf/test.config

@@ -14,13 +14,15 @@
 */
 
 params {
-    config_profile_name        = 'Test profile'
-    config_profile_description = 'Minimal test dataset to check pipeline function'
+    config_profile_name        = 'GitHub FULL test'
+    config_profile_description = 'FULL Test Data for GitHub Actions test'
 
     // Limit resources so that this can run on GitHub Actions
-    max_cpus    = 6
-    max_memory  = '12.GB'
+    max_cpus    = 2
+    max_memory  = '6.GB'
     max_time    = '6.h'
 
-    input       = 'assets/local_testing/nxOscSUBSET.yaml'
+    // Input data
+    input       = "${projectDir}/assets/github_testing/TreeValTinyTest.yaml"
+    outdir      = "TinyTest"
 }

conf/farm_test.config → conf/test_full.config

@@ -20,5 +20,5 @@ params {
     config_profile_description  = "FULL test dataset to check pipeline function, using a current full local dataset"
 
     input                       = "${projectDir}/assets/local_testing/nxOscDF5033.yaml"
-    output                      = nxOscDF5033
+    outdir                      = "SmallTest"
 }

main.nf

@@ -48,7 +48,7 @@ workflow SANGERTOL_TREEVAL_RAPID {
 //
 // WORKFLOWS: Execute named workflow for the pipeline
 //
-workflow FULL {
+workflow {
         SANGERTOL_TREEVAL ()
 }
 

nextflow.config

@@ -60,6 +60,7 @@ try {
 
 
 profiles {
+    cleanup { cleanup = true }
     debug {
         dumpHashes             = true
         process.beforeScript   = 'echo $HOSTNAME'
@@ -151,9 +152,8 @@ profiles {
         executor.memory        = 60.GB
     }
 
-    farm_test               {   includeConfig 'conf/farm_test.config'         }
-    github_test             {   includeConfig 'conf/github_test.config'       }
-    local_github_test       {   includeConfig 'conf/local_github_test.config' }
+    test                    {   includeConfig 'conf/test.config'         }
+    test_full               {   includeConfig 'conf/test_full.config'         }
 }