Merge pull request #18 from pdimens/dev

begin release of 0.4.0

.github/workflows/align.yml

@@ -7,12 +7,13 @@ on:
     paths:
       - 'rules/align-**.smk'
       - '**align.py'
+      - '**helperfunctions.py'
       - 'reports/reportBwaGencov.Rmd'
       - 'reports/reportEmaGencov.Rmd'
       - 'reports/reportEmaCount.Rmd'
       - 'reports/reportBxCount.Rmd'
       - 'reports/reportBxStats.Rmd'
-      - 'reports/ilities/bxStats.py'
+      - 'utilities/bxStats.py'
       - 'utilities/countBX.py'
       - 'utilities/makewindows.py'
       - 'test/fastq/**'
@@ -24,20 +25,18 @@ on:
     paths:
       - 'rules/align-**.smk'
       - '**align.py'
+      - '**helperfunctions.py'
       - 'reports/reportBwaGencov.Rmd'
       - 'reports/reportEmaGencov.Rmd'
       - 'reports/reportEmaCount.Rmd'
       - 'reports/reportBxCount.Rmd'
       - 'reports/reportBxStats.Rmd'
-      - 'reports/ilities/bxStats.py'
+      - 'utilities/bxStats.py'
       - 'utilities/countBX.py'
       - 'utilities/makewindows.py'
       - 'test/fastq/**'
       - '**align.yml'
 
-  schedule:
-    - cron: '0 0 * * *'
-
 env:
   CACHE_NUMBER: 0  # increase to reset cache manually
 
@@ -70,9 +69,9 @@ jobs:
 
       - name: align bwa
         shell: micromamba-shell {0}
-        run: harpy align --method bwa -d test/fastq -g test/genome/genome.fasta.gz -s "--show-failed-logs"
+        run: harpy align bwa -d test/fastq -g test/genome/genome.fasta.gz -s "--show-failed-logs"
         continue-on-error: true
 
       - name: align ema
         shell: micromamba-shell {0}
-        run: harpy align --method ema --ema-bins 20 -d test/fastq -g test/genome/genome.fasta.gz -s "--show-failed-logs"
+        run: harpy align ema --ema-bins 20 -d test/fastq -g test/genome/genome.fasta.gz -s "--show-failed-logs"

.github/workflows/demux.yml

@@ -7,6 +7,7 @@ on:
     paths:
       - 'rules/demultiplex**.smk'
       - '**demultiplex.py'
+      - '**helperfunctions.py'
       - '**demuxGen1'
       - '**demux.yml'
       - 'BC_files.py'
@@ -17,14 +18,13 @@ on:
     - 'dev'
     paths:
       - 'rules/demultiplex**.smk'
+      - '**helperfunctions.py'
       - '**demultiplex.py'
       - '**demuxGen1'
       - '**demux.yml'
       - 'BC_files.py'
       - 'test/data/demux/**'
 
-  schedule:
-    - cron: '0 0 * * *'
 
 env:
   CACHE_NUMBER: 0  # increase to reset cache manually
@@ -53,6 +53,6 @@ jobs:
           misc/buildforCI.sh
         shell: micromamba-shell {0}
 
-      - name: Test harpy demux
-        run: harpy demultiplex --file test/demux/Undetermined_S0_L004_I1_001.fastq.gz --samplesheet test/demux/samples.schema
+      - name: harpy demultiplex
+        run: harpy demultiplex gen1 --file test/demux/Undetermined_S0_L004_I1_001.fastq.gz --samplesheet test/demux/samples.schema
         shell: micromamba-shell {0}

.github/workflows/extras.yml

@@ -0,0 +1,59 @@
+name: Test Extras
+
+on:
+  push:
+    branches:
+      - 'dev'
+    paths:
+      - '**hpc.py'
+      - '**stitchparams.py'
+      - '**popgroup.py'
+      - '**extras.yml'
+
+  pull_request:
+    branches:
+      - 'dev'
+    paths:
+      - '**hpc.py'
+      - '**stitchparams.py'
+      - '**popgroup.py'
+      - '**extras.yml'
+
+env:
+  CACHE_NUMBER: 0  # increase to reset cache manually
+
+jobs:
+  build:
+    name: linux64
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: setup mamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          init-shell: bash
+          generate-run-shell: true
+          environment-file: misc/harpyenv.yaml
+          cache-environment: true
+          post-cleanup: 'all'
+      - name: Install harpy
+        run: |
+          python3 -m pip install --upgrade build && python3 -m build
+          pip install dist/*.whl
+          misc/buildforCI.sh
+        shell: micromamba-shell {0}
+
+      - name: harpy popgroup
+        run: harpy popgroup -d test/fastq
+        shell: micromamba-shell {0}
+
+      - name: harpy stitchparams
+        run: harpy stitchparams -o params.file
+        shell: micromamba-shell {0}
+
+      - name: harpy hpc
+        run: harpy hpc -o hpc.yml -s slurm
+        shell: micromamba-shell {0}

.github/workflows/impute.yml

@@ -7,6 +7,7 @@ on:
     paths:
       - 'rules/impute.smk'
       - '**impute.py'
+      - '**helperfunctions.py'
       - 'test/bam/**'
       - 'test/vcf/test.bcf'
       - '**impute.yml'
@@ -20,16 +21,14 @@ on:
     paths:
       - 'rules/impute.smk'
       - '**impute.py'
+      - '**helperfunctions.py'
       - 'test/bam/**'
       - 'test/vcf/test.bcf'
       - '**impute.yml'
       - 'reports/reportImpute.Rmd'
       - 'reports/reportStitch.Rmd'
       - 'utilities/stitch_impute.R'
 
-  schedule:
-    - cron: '0 0 * * *'
-
 env:
   CACHE_NUMBER: 0  # increase to reset cache manually
 

.github/workflows/phase.yml

@@ -7,6 +7,7 @@ on:
     paths:
       - 'rules/phase-pop.smk'
       - '**phase.py'
+      - '**helperfunctions.py'
       - 'test/bam/**'
       - 'test/vcf/test.bcf'
       - '**phase.yml'
@@ -20,16 +21,14 @@ on:
     paths:
       - 'rules/phase-pop.smk'
       - '**phase.py'
+      - '**helperfunctions.py'
       - 'test/bam/**'
       - 'test/vcf/test.bcf'
       - '**phase.yml'
       - 'reports/reportHapCut2.Rmd'
       - 'utilities/parsePhaseBlocks.py'
       - 'utilities/summarizeHaplobocks.py'
 
-  schedule:
-    - cron: '0 0 * * *'
-
 env:
   CACHE_NUMBER: 0  # increase to reset cache manually
 

.github/workflows/preflight.yml

@@ -0,0 +1,69 @@
+name: Test Preflight
+
+on:
+  push:
+    branches:
+      - 'dev'
+    paths:
+      - 'rules/preflight-fastq.smk'
+      - 'rules/preflight-bam.smk'
+      - '**preflight.py'
+      - '**helperfunctions.py'
+      - 'test/fastq/**'
+      - 'test/bam/**'
+      - 'utilities/checkBAM.py'
+      - 'utilities/checkFASTQ.py'
+      - 'reports/reportPreflightFastq.Rmd'
+      - 'reports/reportPreflightBam.Rmd'
+      - '**preflight.yml'
+
+  pull_request:
+    branches:
+      - 'dev'
+    paths:
+      - 'rules/preflight-fastq.smk'
+      - 'rules/preflight-bam.smk'
+      - '**preflight.py'
+      - '**helperfunctions.py'
+      - 'test/fastq/**'
+      - 'test/bam/**'
+      - 'utilities/checkBAM.py'
+      - 'utilities/checkFASTQ.py'
+      - 'reports/reportPreflightFastq.Rmd'
+      - 'reports/reportPreflightBam.Rmd'
+      - '**preflight.yml'
+
+env:
+  CACHE_NUMBER: 0  # increase to reset cache manually
+
+jobs:
+  build:
+    name: linux64
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: setup mamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          init-shell: bash
+          generate-run-shell: true
+          environment-file: misc/harpyenv.yaml
+          cache-environment: true
+          post-cleanup: 'all'
+      - name: Install harpy
+        run: |
+          python3 -m pip install --upgrade build && python3 -m build
+          pip install dist/*.whl
+          misc/buildforCI.sh
+        shell: micromamba-shell {0}
+
+      - name: harpy preflight fastq
+        run: harpy preflight fastq -d test/fastq
+        shell: micromamba-shell {0}
+
+      - name: harpy preflight bam
+        run: harpy preflight bam -d test/bam
+        shell: micromamba-shell {0}

.github/workflows/trim.yml → .github/workflows/qc.yml

@@ -1,26 +1,25 @@
-name: Test Trim
+name: Test QC
 
 on:
   push:
     branches:
       - 'dev'
     paths:
-      - 'rules/trim.smk'
-      - '**trim.py'
+      - 'rules/qc.smk'
+      - '**qc.py'
+      - '**helperfunctions.py'
       - 'test/fastq/**'
-      - '**trim.yml'
+      - '**qc.yml'
 
   pull_request:
     branches:
       - 'dev'
     paths:
-      - 'rules/trim.smk'
-      - '**trim.py'
+      - 'rules/qc.smk'
+      - '**qc.py'
+      - '**helperfunctions.py'
       - 'test/fastq/**'
-      - '**trim.yml'
-
-  schedule:
-    - cron: '0 0 * * *'
+      - '**qc.yml'
 
 env:
   CACHE_NUMBER: 0  # increase to reset cache manually
@@ -49,6 +48,6 @@ jobs:
           misc/buildforCI.sh
         shell: micromamba-shell {0}
 
-      - name: Test harpy trim
-        run: harpy trim -d test/fastq
+      - name: harpy qc
+        run: harpy qc -d test/fastq
         shell: micromamba-shell {0}

.github/workflows/variants_snp.yml

@@ -7,7 +7,8 @@ on:
     paths:
       - 'rules/variants-freebayes.smk'
       - 'rules/variants-mpileup.smk'
-      - '**variants_snp.py'
+      - '**snp.py'
+      - '**helperfunctions.py'
       - 'reports/reportBcftools.Rmd'
       - 'utilities/makewindows.py'
       - 'test/bam/**'
@@ -19,15 +20,13 @@ on:
     paths:
       - 'rules/variants-freebayes.smk'
       - 'rules/variants-mpileup.smk'
-      - '**variants_snp.py'
+      - '**snp.py'
+      - '**helperfunctions.py'
       - 'reports/reportBcftools.Rmd'
       - 'utilities/makewindows.py'
       - 'test/bam/**'
       - '**variants_snp.yml'
 
-  schedule:
-    - cron: '0 0 * * *'
-
 env:
   CACHE_NUMBER: 0  # increase to reset cache manually
 
@@ -60,9 +59,9 @@ jobs:
 
       - name: variants mpileup
         shell: micromamba-shell {0}
-        run: harpy variants snp --method mpileup -w 150000 -d test/bam -g test/genome/genome.fasta.gz -s "--show-failed-logs"
+        run: harpy snp mpileup -w 150000 -d test/bam -g test/genome/genome.fasta.gz -s "--show-failed-logs"
         continue-on-error: true
 
       - name: variants freebayes
         shell: micromamba-shell {0}
-        run: harpy variants snp --method freebayes -w 150000 -d test/bam -g test/genome/genome.fasta.gz -s "--show-failed-logs"
+        run: harpy snp freebayes -w 150000 -d test/bam -g test/genome/genome.fasta.gz -s "--show-failed-logs"