Skip to content
Test Harpy Modules

Singularity #72

Sign in to view logs

Singularity

Singularity #72

Workflow file for this run

.github/workflows/tests.yml at 74e085e
name: Test Harpy Modules
on:
push:
branches:
- 'dev'
pull_request:
branches:
- dev
# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
CACHE_NUMBER: 0 # increase to reset cache manually
jobs:
changes:
name: detect file changes
runs-on: ubuntu-latest
# Required permissions
permissions:
pull-requests: read
# Set job outputs to values from filter step
outputs:
preflight: ${{ steps.filter.outputs.preflight }}
demux: ${{ steps.filter.outputs.demux }}
qc: ${{ steps.filter.outputs.qc }}
bwa: ${{ steps.filter.outputs.bwa }}
ema: ${{ steps.filter.outputs.ema }}
minimap: ${{ steps.filter.outputs.minimap }}
mpileup: ${{ steps.filter.outputs.mpileup }}
freebayes: ${{ steps.filter.outputs.freebayes }}
leviathan: ${{ steps.filter.outputs.leviathan }}
naibr: ${{ steps.filter.outputs.naibr }}
impute: ${{ steps.filter.outputs.impute }}
phase: ${{ steps.filter.outputs.phase }}
simvars: ${{ steps.filter.outputs.simvars }}
simreads: ${{ steps.filter.outputs.simreads }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: check which files changed
uses: dorny/paths-filter@v3
id: filter
with:
base: 'dev'
filters: .github/filters.yml
pkgbuild:
name: harpy installation
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: pull image
shell: micromamba-shell {0}
run: snakemake -s resources/container/pullcontainerCI.smk --sdm conda apptainer -q
- name: upload singularity
uses: actions/upload-artifact@v4
with:
name: deps-image
path: .snakemake/singularity
dmux_gen1:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.demux == 'true' && needs.pkgbuild.result == 'success' }}
name: test demux gen1
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: Download a single artifact
uses: actions/download-artifact@v4
with:
name: deps-image
- name: harpy demultiplex
shell: micromamba-shell {0}
run: harpy demultiplex gen1 --schema test/demux/samples.schema test/demux/Undetermined_S0_L004_R* test/demux/Undetermined_S0_L004_I*
preflight:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.preflight == 'true' && needs.pkgbuild.result == 'success' }}
name: test preflight
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: test preflight fastq
shell: micromamba-shell {0}
run: harpy preflight fastq test/fastq
- name: test preflight bam
if: always()
shell: micromamba-shell {0}
run: harpy preflight bam test/bam
qc:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.qc == 'true' && needs.pkgbuild.result == 'success' }}
name: test qc
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: harpy qc
shell: micromamba-shell {0}
run: harpy qc -x "--trim_poly_g" --snakemake "--show-failed-logs" test/fastq
- name: harpy qc skip adapter trimming
if: always()
shell: micromamba-shell {0}
run: harpy qc -a --snakemake "--show-failed-logs" test/fastq
bwa:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.bwa == 'true' && needs.pkgbuild.result == 'success' }}
name: test align BWA
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: test bwa
shell: micromamba-shell {0}
run: harpy align bwa -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "-A 2" test/fastq
ema:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.ema == 'true' && needs.pkgbuild.result == 'success' }}
name: test align EMA
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: test ema
shell: micromamba-shell {0}
run: harpy align ema --ema-bins 150 -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "-d" test/fastq
minimap:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.minimap == 'true' && needs.pkgbuild.result == 'success' }}
name: test align minimap
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: test minimap
shell: micromamba-shell {0}
run: harpy align minimap -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "--seed 13" test/fastq
mpileup:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.mpileup == 'true' && needs.pkgbuild.result == 'success' }}
name: test mpileup
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: snp mpileup
shell: micromamba-shell {0}
run: harpy snp mpileup -r test/positions.bed -g test/genome/genome.fasta.gz -x "--ignore-RG" test/bam
- name: snp mpileup-pop
shell: micromamba-shell {0}
run: harpy snp mpileup -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups --snakemake "--show-failed-logs" test/bam
freebayes:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.freebayes == 'true' && needs.pkgbuild.result == 'success' }}
name: test freebayes
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: snp freebayes
shell: micromamba-shell {0}
run: harpy snp freebayes -r test/positions.bed -g test/genome/genome.fasta.gz -x "-g 200" test/bam
- name: snp freebayes-pop
shell: micromamba-shell {0}
run: harpy snp freebayes -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups --snakemake "--show-failed-logs" test/bam
impute:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.impute == 'true' && needs.pkgbuild.result == 'success' }}
name: test impute
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: impute
shell: micromamba-shell {0}
run: harpy impute --vcf test/vcf/test.bcf -p test/stitch.params --snakemake "--show-failed-logs" test/bam
- name: impute from vcf
shell: micromamba-shell {0}
if: always()
run: harpy impute --vcf-samples -o vcfImpute --vcf test/vcf/test.bcf -p test/stitch.params --snakemake "--show-failed-logs" test/bam
phase:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.phase == 'true' && needs.pkgbuild.result == 'success' }}
name: test phase
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: phase
shell: micromamba-shell {0}
run: harpy phase --vcf test/vcf/test.bcf --snakemake "--show-failed-logs" -x "--max_iter 10001" test/bam
- name: phase with indels
shell: micromamba-shell {0}
if: always()
run: harpy phase --vcf test/vcf/test.bcf -o phaseindel -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" test/bam
- name: phase from vcf
shell: micromamba-shell {0}
if: always()
run: harpy phase --vcf-samples -o phasevcf --vcf test/vcf/test.bcf --snakemake "--show-failed-logs" test/bam
leviathan:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.leviathan == 'true' && needs.pkgbuild.result == 'success' }}
name: test sv leviathan
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: leviathan
shell: micromamba-shell {0}
run: harpy sv leviathan -n 100 -b 1 -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "-M 2002" test/bam
continue-on-error: true
- name: leviathan-pop
if: always()
shell: micromamba-shell {0}
run: harpy sv leviathan -n 100 -b 1 -g test/genome/genome.fasta.gz -o SV/leviathanpop -p test/samples.groups --snakemake "--show-failed-logs" test/bam
naibr:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.naibr == 'true' && needs.pkgbuild.result == 'success' }}
name: test sv naibr
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: naibr
shell: micromamba-shell {0}
run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/naibr --snakemake "--show-failed-logs" -x "-min_sv 5000" test/bam_phased && rm -r Genome
- name: naibr pop
if: always()
shell: micromamba-shell {0}
run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/pop -p test/samples.groups --snakemake "--show-failed-logs" test/bam_phased && rm -r Genome
- name: naibr with phasing
if: always()
shell: micromamba-shell {0}
run: |
harpy sv naibr -g test/genome/genome.fasta.gz -o SV/phase -v test/vcf/test.phased.bcf --snakemake "--show-failed-logs" test/bam && rm -r Genome
- name: naibr pop with phasing
if: always()
shell: micromamba-shell {0}
run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/phasepop -v test/vcf/test.phased.bcf -p test/samples.groups --snakemake "--show-failed-logs" test/bam && rm -r Genome
simulate_variants:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.simvars == 'true' && needs.pkgbuild.result == 'success' }}
name: test simulate variants
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
shell: micromamba-shell {0}
- name: simulate random snps/indels
shell: micromamba-shell {0}
run: |
harpy simulate snpindel --snp-count 10 --indel-count 10 -z 0.5 --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
harpy simulate snpindel --prefix Simulate/snpvcf --snp-vcf Simulate/snpindel/sim.snpindel.snp.hap1.vcf --indel-vcf Simulate/snpindel/sim.snpindel.indel.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
- name: simulate inversions
shell: micromamba-shell {0}
if: always()
run: |
harpy simulate inversion --count 10 -z 0.5 test/genome/genome.fasta.gz
harpy simulate inversion --prefix Simulate/invvcf --vcf Simulate/inversion/sim.inversion.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
- name: simulate cnv
shell: micromamba-shell {0}
if: always()
run: |
harpy simulate cnv --count 10 -z 0.5 test/genome/genome.fasta.gz
harpy simulate cnv --prefix Simulate/cnvvcf --vcf Simulate/cnv/sim.cnv.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
- name: simulate translocations
shell: micromamba-shell {0}
if: always()
run: |
harpy simulate translocation --count 10 -z 0.5 test/genome/genome.fasta.gz
harpy simulate translocation --prefix Simulate/transvcf --vcf Simulate/translocation/sim.translocation.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
simulate_linkedreads:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.simreads == 'true' && needs.pkgbuild.result == 'success' }}
name: test simulate linkedreads
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: simulate linked reads
shell: micromamba-shell {0}
run: harpy simulate linkedreads --snakemake "--show-failed-logs" -t 4 -n 2 -l 100 -p 50 test/genome/genome.fasta.gz test/genome/genome2.fasta.gz
extras:
needs: [changes, pkgbuild]
if: ${{ needs.changes.outputs.extras == 'true' && needs.pkgbuild.result == 'success' }}
name: test harpy extras
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
with:
fetch-depth: 1
- name: setup mamba
uses: mamba-org/setup-micromamba@v1
env:
ACTIONS_STEP_DEBUG: true
with:
init-shell: bash
generate-run-shell: true
environment-file: resources/harpy.yaml
cache-environment: true
post-cleanup: 'all'
log-level: error
- name: Install harpy
shell: micromamba-shell {0}
run: |
python3 -m pip install --upgrade build && python3 -m build
pip install dist/*.whl
resources/buildforCI.sh
- name: Clear space
run: rm -rf /opt/hostedtoolcache
- name: harpy stitchparams
shell: micromamba-shell {0}
run: harpy stitchparams -o params.file
- name: harpy popgroup
shell: micromamba-shell {0}
run: harpy popgroup test/fastq