tests.yml

name: Test Harpy Modules

on:
  push:
    branches:
      - 'dev'
  pull_request:
    branches:
      - dev

# Cancel in progress workflows on pull_requests.
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  CACHE_NUMBER: 0  # increase to reset cache manually

jobs:
  changes:
    name: detect file changes
    runs-on: ubuntu-latest
      # Required permissions
    permissions:
      pull-requests: read
    # Set job outputs to values from filter step
    outputs:
      preflight: ${{ steps.filter.outputs.preflight }}
      demux: ${{ steps.filter.outputs.demux }}
      qc: ${{ steps.filter.outputs.qc }}
      bwa: ${{ steps.filter.outputs.bwa }}
      ema: ${{ steps.filter.outputs.ema }}
      minimap: ${{ steps.filter.outputs.minimap }}
      mpileup: ${{ steps.filter.outputs.mpileup }}
      freebayes: ${{ steps.filter.outputs.freebayes }}
      leviathan: ${{ steps.filter.outputs.leviathan }}
      naibr: ${{ steps.filter.outputs.naibr }}
      impute: ${{ steps.filter.outputs.impute }}
      phase: ${{ steps.filter.outputs.phase }}
      simvars: ${{ steps.filter.outputs.simvars }}
      simreads: ${{ steps.filter.outputs.simreads }}
    steps:
      - name: checkout
        uses: actions/checkout@v4
      - name: check which files changed
        uses: dorny/paths-filter@v3
        id: filter
        with:
          base: 'dev'
          filters: .github/filters.yml

  pkgbuild:
    name: harpy installation
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: pull image
        shell: micromamba-shell {0}
        run: snakemake -s resources/container/pullcontainerCI.smk --sdm conda apptainer -q
      - name: upload singularity
        uses: actions/upload-artifact@v4
        with:
          name: deps-image
          path: .snakemake/singularity

  dmux_gen1:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.demux == 'true' && needs.pkgbuild.result == 'success' }}
    name: test demux gen1
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: Download a single artifact
        uses: actions/download-artifact@v4
        with:
          name: deps-image
      - name: harpy demultiplex
        shell: micromamba-shell {0}
        run: harpy demultiplex gen1  --schema test/demux/samples.schema test/demux/Undetermined_S0_L004_R* test/demux/Undetermined_S0_L004_I*

  preflight:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.preflight == 'true' && needs.pkgbuild.result == 'success' }}
    name: test preflight
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: test preflight fastq
        shell: micromamba-shell {0}
        run: harpy preflight fastq test/fastq
      - name: test preflight bam
        if: always()
        shell: micromamba-shell {0}
        run: harpy preflight bam test/bam

  qc:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.qc == 'true' && needs.pkgbuild.result == 'success' }}
    name: test qc
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: harpy qc
        shell: micromamba-shell {0}
        run: harpy qc -x "--trim_poly_g"  --snakemake "--show-failed-logs" test/fastq
      - name: harpy qc skip adapter trimming
        if: always()
        shell: micromamba-shell {0}
        run: harpy qc -a --snakemake "--show-failed-logs" test/fastq
  
  bwa:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.bwa == 'true' && needs.pkgbuild.result == 'success' }}
    name: test align BWA
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: test bwa
        shell: micromamba-shell {0}
        run: harpy align bwa -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "-A 2" test/fastq
  
  ema:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.ema == 'true' && needs.pkgbuild.result == 'success' }}
    name: test align EMA
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: test ema
        shell: micromamba-shell {0}
        run: harpy align ema --ema-bins 150 -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "-d" test/fastq

  minimap:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.minimap == 'true' && needs.pkgbuild.result == 'success' }}
    name: test align minimap
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: test minimap
        shell: micromamba-shell {0}
        run: harpy align minimap -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "--seed 13" test/fastq

  mpileup:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.mpileup == 'true' && needs.pkgbuild.result == 'success' }}
    name: test mpileup
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: snp mpileup
        shell: micromamba-shell {0}
        run: harpy snp mpileup -r test/positions.bed -g test/genome/genome.fasta.gz -x "--ignore-RG" test/bam
      - name: snp mpileup-pop
        shell: micromamba-shell {0}
        run: harpy snp mpileup -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups --snakemake "--show-failed-logs" test/bam
  
  freebayes:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.freebayes == 'true' && needs.pkgbuild.result == 'success' }}
    name: test freebayes
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: snp freebayes
        shell: micromamba-shell {0}
        run: harpy snp freebayes -r test/positions.bed -g test/genome/genome.fasta.gz -x "-g 200" test/bam
      - name: snp freebayes-pop
        shell: micromamba-shell {0}
        run: harpy snp freebayes -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups --snakemake "--show-failed-logs" test/bam

  impute:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.impute == 'true' && needs.pkgbuild.result == 'success' }}
    name: test impute
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: impute
        shell: micromamba-shell {0}
        run: harpy impute --vcf test/vcf/test.bcf -p test/stitch.params --snakemake "--show-failed-logs" test/bam
      - name: impute from vcf
        shell: micromamba-shell {0}
        if: always()
        run: harpy impute --vcf-samples -o vcfImpute --vcf test/vcf/test.bcf -p test/stitch.params --snakemake "--show-failed-logs" test/bam

  phase:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.phase == 'true' && needs.pkgbuild.result == 'success' }}
    name: test phase
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: phase
        shell: micromamba-shell {0}
        run: harpy phase --vcf test/vcf/test.bcf --snakemake "--show-failed-logs" -x "--max_iter 10001" test/bam
      - name: phase with indels
        shell: micromamba-shell {0}
        if: always()
        run: harpy phase --vcf test/vcf/test.bcf -o phaseindel -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" test/bam
      - name: phase from vcf
        shell: micromamba-shell {0}
        if: always()
        run: harpy phase --vcf-samples -o phasevcf --vcf test/vcf/test.bcf --snakemake "--show-failed-logs" test/bam

  leviathan:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.leviathan == 'true' && needs.pkgbuild.result == 'success' }}
    name: test sv leviathan
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: leviathan
        shell: micromamba-shell {0}
        run: harpy sv leviathan -n 100 -b 1 -g test/genome/genome.fasta.gz --snakemake "--show-failed-logs" -x "-M 2002" test/bam
        continue-on-error: true
      - name: leviathan-pop
        if: always()
        shell: micromamba-shell {0}
        run: harpy sv leviathan -n 100 -b 1 -g test/genome/genome.fasta.gz -o SV/leviathanpop -p test/samples.groups --snakemake "--show-failed-logs" test/bam

  naibr:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.naibr == 'true' && needs.pkgbuild.result == 'success' }}
    name: test sv naibr
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: naibr
        shell: micromamba-shell {0}
        run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/naibr --snakemake "--show-failed-logs" -x "-min_sv 5000" test/bam_phased && rm -r Genome
      - name: naibr pop
        if: always()
        shell: micromamba-shell {0}
        run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/pop -p test/samples.groups --snakemake "--show-failed-logs" test/bam_phased && rm -r Genome
      - name: naibr with phasing
        if: always()
        shell: micromamba-shell {0}
        run: |
          harpy sv naibr -g test/genome/genome.fasta.gz -o SV/phase -v test/vcf/test.phased.bcf --snakemake "--show-failed-logs" test/bam && rm -r Genome
      - name: naibr pop with phasing
        if: always()
        shell: micromamba-shell {0}
        run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/phasepop -v test/vcf/test.phased.bcf -p test/samples.groups --snakemake "--show-failed-logs" test/bam && rm -r Genome


  simulate_variants:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.simvars == 'true' && needs.pkgbuild.result == 'success' }}
    name: test simulate variants
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
        shell: micromamba-shell {0}
      - name: simulate random snps/indels
        shell: micromamba-shell {0}
        run: |
          harpy simulate snpindel --snp-count 10 --indel-count 10 -z 0.5 --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
          harpy simulate snpindel --prefix Simulate/snpvcf --snp-vcf Simulate/snpindel/sim.snpindel.snp.hap1.vcf --indel-vcf Simulate/snpindel/sim.snpindel.indel.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
      - name: simulate inversions
        shell: micromamba-shell {0}
        if: always()
        run: |
          harpy simulate inversion --count 10 -z 0.5 test/genome/genome.fasta.gz
          harpy simulate inversion --prefix Simulate/invvcf --vcf Simulate/inversion/sim.inversion.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
      - name: simulate cnv
        shell: micromamba-shell {0}
        if: always()
        run: |
          harpy simulate cnv --count 10 -z 0.5 test/genome/genome.fasta.gz
          harpy simulate cnv --prefix Simulate/cnvvcf --vcf Simulate/cnv/sim.cnv.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz
      - name: simulate translocations
        shell: micromamba-shell {0}
        if: always()
        run: |
          harpy simulate translocation --count 10 -z 0.5 test/genome/genome.fasta.gz
          harpy simulate translocation --prefix Simulate/transvcf --vcf Simulate/translocation/sim.translocation.hap1.vcf --snakemake "--show-failed-logs" test/genome/genome.fasta.gz

  simulate_linkedreads:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.simreads == 'true' && needs.pkgbuild.result == 'success' }}
    name: test simulate linkedreads
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: simulate linked reads
        shell: micromamba-shell {0}
        run: harpy simulate linkedreads --snakemake "--show-failed-logs" -t 4 -n 2  -l 100 -p 50 test/genome/genome.fasta.gz test/genome/genome2.fasta.gz

  extras:
    needs: [changes, pkgbuild]
    if: ${{ needs.changes.outputs.extras == 'true' && needs.pkgbuild.result == 'success' }}
    name: test harpy extras
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: setup mamba
        uses: mamba-org/setup-micromamba@v1
        env:
          ACTIONS_STEP_DEBUG: true
        with:
          init-shell: bash
          generate-run-shell: true
          environment-file: resources/harpy.yaml
          cache-environment: true
          post-cleanup: 'all'
          log-level: error
      - name: Install harpy
        shell: micromamba-shell {0}
        run: |
          python3 -m pip install --upgrade build && python3 -m build
          pip install dist/*.whl
          resources/buildforCI.sh
      - name: Clear space
        run: rm -rf /opt/hostedtoolcache
      - name: harpy stitchparams
        shell: micromamba-shell {0}
        run: harpy stitchparams -o params.file
      - name: harpy popgroup
        shell: micromamba-shell {0}
        run: harpy popgroup test/fastq