Ncbench #96
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: NCBench Zenodo Upload | |
| # This workflow can be triggered manually with the GitHub actions workflow dispatch button. | |
| on: | |
| # upload can only be triggered manually for now | |
| workflow_dispatch: | |
| inputs: | |
| # TODO: add in other test profiles | |
| germline_default_agilent: | |
| description: "Trigger NCBench upload" | |
| type: boolean | |
| default: true | |
| pull_request: # TODO: remove once it works | |
| jobs: | |
| ncbench-upload: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # TODO: Add in other test profiles | |
| - profile: germline_default_agilent | |
| enabled: ${{ ( github.repository == 'nf-core/sarek' ) && ( github.event_name != 'workflow_dispatch' || inputs.germline_default_agilent ) }} | |
| steps: | |
| - name: Download pipeline | |
| uses: actions/checkout@v4 | |
| - name: Extract Versions | |
| id: extract_versions | |
| run: | | |
| PIPELINE_VERSION=$(awk -F"'" '/manifest {/,/version/ {if(/version/) print $2}' nextflow.config) | |
| echo "PIPELINE_VERSION=${PIPELINE_VERSION}" >> $GITHUB_ENV | |
| FASTP_VERSION=$(awk '/- bioconda::fastp=/ {gsub(/.*=/, "", $NF); gsub(/'\''/, "", $NF); print $NF}' modules/nf-core/fastp/environment.yml) | |
| echo "FASTP_VERSION=${FASTP_VERSION}" >> $GITHUB_ENV | |
| BWA_VERSION=$(awk '/- bwa=/ {gsub(/.*=/, "", $NF); gsub(/'\''/, "", $NF); print $NF}' modules/nf-core/bwa/mem/environment.yml) | |
| echo "BWA_VERSION=${BWA_VERSION}" >> $GITHUB_ENV | |
| GATK_VERSION=$(awk '/- bioconda::gatk4=/ {gsub(/.*=/, "", $NF); gsub(/'\''/, "", $NF); print $NF}' modules/nf-core/gatk4/baserecalibrator/environment.yml) | |
| echo "GATK_VERSION=${GATK_VERSION}" >> $GITHUB_ENV | |
| STRELKA_VERSION=$(awk '/- bioconda::strelka=/ {gsub(/.*=/, "", $NF); gsub(/'\''/, "", $NF); print $NF}' modules/nf-core/strelka/germline/environment.yml) | |
| echo "STRELKA_VERSION=${STRELKA_VERSION}" >> $GITHUB_ENV | |
| - name: Download files from AWS | |
| uses: keithweaver/[email protected] | |
| if: ${{ matrix.enabled }} | |
| with: | |
| # TODO: fix path to agilent test path | |
| source: s3://nf-core-awsmegatests/sarek/results-test-59026dc07633edb83aab3bfb2f65f79db38437a1/test_full/variant_calling | |
| command: sync | |
| aws_region: eu-west-1 | |
| destination: ./variant_calling | |
| flags: --no-sign-request --include ".vcf.gz" --exclude "g.vcf.gz" | |
| # | |
| # - name: list files | |
| # run: | | |
| # ls variant_calling/strelka/HCC1395N/ | |
| # | |
| - name: Install requests | |
| run: pip install requests # version pinning? | |
| - name: Upload to zenodo | |
| env: | |
| ACCESS_TOKEN: ${{ secrets.NCBENCH_CI_TOKEN_SANDBOX }} # TODO ${{ secrets.ZENODO_DEPOSIT }} | |
| run: | | |
| python .github/workflows/upload.py | |
| echo $DEPOSITION_ID | |
| # update ncbench config yml | |
| - name: NCBench set config | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.NCBENCH_CI_TOKEN }} | |
| repository: FriederikeHanssen/ncbench-workflow | |
| path: ncbench-workflow/ | |
| ref: main | |
| - name: Add entry to config | |
| uses: mikefarah/yq@master | |
| with: | |
| # TODO: add in values for all callers, ensure proper extension for the datasets are used. | |
| # TODO check if trimming should be enabled actually | |
| cmd: for reads in 75 200; do | |
| yq --inplace ' | |
| with(.variant-calls.nf-core-sarek-${{ env.PIPELINE_VERSION }}-strelka-agilent-${reads}M.labels; | |
| .site = "nf-core" | | |
| .pipeline = "nf-core/sarek v${{ env.PIPELINE_VERSION }}" | | |
| .trimming = "FastP v${{ env.FASTP_VERSION }}" | | |
| .read-mapping = "bwa mem v${{ env.BWA_VERSION }}" | | |
| .base-quality-recalibration = "gatk4 v${{ env.GATK_VERSION }}" | | |
| .realignment = "none" | | |
| .variant-detection = "strelka2 v${{ env.STRELKA_VERSION }}" | | |
| .genotyping = "none" | | |
| .reads = "${reads}M" ) | | |
| with(.variant-calls.nf-core-sarek-${{ env.PIPELINE_VERSION }}-strelka-agilent-${reads}M.subcategory; | |
| . = "NA12878-agilent" ) | | |
| with(.variant-calls.nf-core-sarek-${{ env.PIPELINE_VERSION }}-strelka-agilent-${reads}M.zenodo; | |
| .deposition = $DEPOSITION_ID | | |
| .filename = "TODO get here proper file names" ) | | |
| with(.variant-calls.nf-core-sarek-${{ env.PIPELINE_VERSION }}-strelka-agilent-${reads}M.benchmark; | |
| . = "giab-NA12878-agilent-${reads}M" ) | | |
| with(.variant-calls.nf-core-sarek-${{ env.PIPELINE_VERSION }}-strelka-agilent-${reads}M.rename-contigs; | |
| . = "resources/rename-contigs/ucsc-to-ensembl.txt" ) | |
| ' ncbench-workflow/config/config.yaml | |
| done | |
| # TODO: enable using secrets here | |
| - name: Commit files | |
| run: | | |
| cd ncbench-workflow | |
| git config --global user.email "[email protected]" | |
| git config --global user.name "FriederikeHanssen" | |
| git checkout -b sarek_${{ env.PIPELINE_VERSION }} | |
| git add config/config.yaml | |
| git commit -a -m "Add changes" | |
| git push origin sarek_${{ env.PIPELINE_VERSION }} |