Merge pull request #102 from pdimens/deconvolute

Deconvolute

.github/filters.yml

@@ -14,33 +14,38 @@ preflight: &preflight
   - *common
   - *container
   - 'src/harpy/preflight.py'
-  - 'src/harpy/rules/preflight-fastq.smk'
-  - 'src/harpy/rules/preflight-bam.smk'
+  - 'src/harpy/snakefiles/preflight-fastq.smk'
+  - 'src/harpy/snakefiles/preflight-bam.smk'
   - 'test/fastq/**'
   - 'test/bam/**'
   - 'src/harpy/bin/checkBAM.py'
   - 'src/harpy/bin/checkFASTQ.py'
   - 'src/harpy/reports/PreflightFastq.Rmd'
   - 'src/harpy/reports/PreflightBam.Rmd'
+deconvolve: &deconvolve
+  - *common
+  - *container
+  - 'src/harpy/deconvolve.py'
+  - 'src/harpy/snakefiles/deconvolve.smk'
 demux: &demux
   - *common
   - *container
   - 'src/harpy/demultiplex.py'
-  - 'src/harpy/rules/demultiplex**.smk'
+  - 'src/harpy/snakefiles/demultiplex**.smk'
   - '**demuxGen1'
   - 'bcFiles.py'
   - 'test/data/demux/**'
 qc: &qc
   - *common
   - *container
   - 'src/harpy/qc.py'
-  - 'src/harpy/rules/qc.smk'
+  - 'src/harpy/snakefiles/qc.smk'
   - 'test/fastq/**'
 bwa: &bwa
   - *common
   - *container
   - 'src/harpy/align.py'
-  - 'src/harpy/rules/align-bwa.smk'
+  - 'src/harpy/snakefiles/align-bwa.smk'
   - 'src/harpy/reports/AlignStats.Rmd'
   - 'src/harpy/reports/BxCount.Rmd'
   - 'src/harpy/bin/bxStats.py'
@@ -52,7 +57,7 @@ ema: &ema
   - *common
   - *container
   - 'src/harpy/align.py'
-  - 'src/harpy/rules/align-ema.smk'
+  - 'src/harpy/snakefiles/align-ema.smk'
   - 'src/harpy/reports/AlignStats.Rmd'
   - 'src/harpy/reports/BxCount.Rmd'
   - 'src/harpy/bin/bxStats.py'
@@ -63,7 +68,7 @@ strobealign: &strobealign
   - *common
   - *container
   - 'src/harpy/align.py'
-  - 'src/harpy/rules/align-strobealign.smk'
+  - 'src/harpy/snakefiles/align-strobealign.smk'
   - 'src/harpy/reports/AlignStats.Rmd'
   - 'src/harpy/reports/BxCount.Rmd'
   - 'src/harpy/bin/assignMI.py'
@@ -75,23 +80,23 @@ mpileup: &mpileup
   - *common
   - *container
   - 'src/harpy/snp.py'
-  - 'src/harpy/rules/snp-mpileup.smk'
+  - 'src/harpy/snakefiles/snp-mpileup.smk'
   - 'src/harpy/reports/BcftoolsStats.Rmd'
   - 'src/harpy/bin/makewindows.py'
   - 'test/bam/**'
 freebayes: &freebayes
   - *common
   - *container
   - 'src/harpy/snp.py'
-  - 'src/harpy/rules/snp-freebayes.smk'
+  - 'src/harpy/snakefiles/snp-freebayes.smk'
   - 'src/harpy/reports/BcftoolsStats.Rmd'
   - 'src/harpy/bin/makewindows.py'
   - 'test/bam/**'
 impute: &impute
   - *common
   - *container
   - 'src/harpy/impute.py'
-  - 'src/harpy/rules/impute.smk'
+  - 'src/harpy/snakefiles/impute.smk'
   - 'test/bam/**'
   - 'test/vcf/test.bcf'
   - 'src/harpy/reports/Impute.Rmd'
@@ -101,14 +106,14 @@ leviathan: &leviathan
   - *common
   - *container
   - 'src/harpy/sv.py'
-  - 'src/harpy/rules/sv-leviathan**.smk'
+  - 'src/harpy/snakefiles/sv-leviathan**.smk'
   - 'test/bam/**'
   - 'src/harpy/reports/Leviathan**.Rmd'
 naibr: &naibr
   - *common
   - *container
   - 'src/harpy/sv.py'
-  - 'src/harpy/rules/sv-naibr**.smk'
+  - 'src/harpy/snakefiles/sv-naibr**.smk'
   - 'test/bam/**'
   - 'test/bam_phased/**'
   - 'test/vcf/test.phased.bcf'
@@ -118,7 +123,7 @@ phase: &phase
   - *common
   - *container
   - 'src/harpy/phase.py'
-  - 'src/harpy/rules/phase.smk'
+  - 'src/harpy/snakefiles/phase.smk'
   - 'test/bam/**'
   - 'test/vcf/test.bcf'
   - 'src/harpy/reports/HapCut2.Rmd'
@@ -127,15 +132,15 @@ simvars: &simvars
   - *common
   - *container
   - 'src/harpy/simulatevariants.py'
-  - 'simulate-snpindel.smk'
-  - 'simulate-variants.smk'
+  - 'src/harpy/snakefiles/simulate-snpindel.smk'
+  - 'src/harpy/snakefiles/simulate-variants.smk'
   - 'test/vcf/test.bcf'
   - 'src/harpy/scripts/simuG.pl'
 simreads: &simreads
   - *common
   - *container
   - 'src/harpy/simulatelinkedreads.py'
-  - 'simulate-linkedreads.smk'
+  - 'src/harpy/snakefiles/simulate-linkedreads.smk'
   - 'test/genome**gz'
   - 'extractReads.cpp'
   - 'src/harpy/bin/10xtoHaplotag.py'
@@ -147,6 +152,7 @@ other: &other
 modules:
   - *common
   - *demux
+  - *deconvolve
   - *preflight
   - *qc
   - *bwa
@@ -160,4 +166,4 @@ modules:
   - *naibr
   - *simvars
   - *simreads
-  - *other
+  - *other

.github/workflows/createrelease.yml

@@ -65,6 +65,11 @@ jobs:
           for i in src/harpy/snakefiles/*.smk; do
             sed -i "s/harpy\:latest/harpy\:${{ github.ref_name }}/g" $i
           done
+      - name: Bump Harpy Version
+        # this removes the :latest tag and replaces with versioned container
+        run: |
+          sed -i "s/0\.0\.0/${{ github.ref_name }}/g" src/harpy/__main__.py
+          sed -i "s/0\.0\.0/${{ github.ref_name }}/g" pyproject.toml         
       - name: Build project
         # This builds the release tarball, stripped of unneccessary things
         run: |

.github/workflows/tests.yml

@@ -27,6 +27,7 @@ jobs:
     # Set job outputs to values from filter step
     outputs:
       preflight: ${{ steps.filter.outputs.preflight }}
+      deconvolve: ${{ steps.filter.outputs.deconvolve }}
       demux: ${{ steps.filter.outputs.demux }}
       qc: ${{ steps.filter.outputs.qc }}
       bwa: ${{ steps.filter.outputs.bwa }}
@@ -228,11 +229,46 @@ jobs:
       - name: harpy qc
         shell: micromamba-shell {0}
         run: harpy qc -x "--trim_poly_g"  --snakemake "--show-failed-logs" test/fastq
-      - name: harpy qc skip adapter trimming
-        if: always()
+      - name: harpy qc all options
         shell: micromamba-shell {0}
-        run: harpy qc -a --snakemake "--show-failed-logs" test/fastq
-
+        run: harpy qc -a -d -c --snakemake "--show-failed-logs" test/fastq
+  deconvolve:
+    needs: [changes, pkgbuild]
+    if: ${{ needs.changes.outputs.deconvolve == 'true' && needs.pkgbuild.result == 'success' }}
+    name: deconvolve
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Setup Mamba
+        uses: mamba-org/setup-micromamba@v1
+        env:
+          ACTIONS_STEP_DEBUG: true
+        with:
+          init-shell: bash
+          generate-run-shell: true
+          environment-file: resources/harpy.yaml
+          cache-environment: true
+          post-cleanup: 'all'
+          log-level: error
+      - name: Install Harpy
+        shell: micromamba-shell {0}
+        run: |
+          python3 -m pip install --upgrade build && python3 -m build
+          pip install dist/*.whl
+          resources/buildforCI.sh
+      - name: Clear Space
+        run: rm -rf /opt/hostedtoolcache && mkdir -p .snakemake/singularity
+      - name: Download Singularity Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: deps-image
+          path: .snakemake/singularity
+      - name: harpy deconvolve
+        shell: micromamba-shell {0}
+        run: harpy deconvolve --snakemake "--show-failed-logs" test/fastq
   bwa:
     needs: [changes, pkgbuild]
     if: ${{ needs.changes.outputs.bwa == 'true' && needs.pkgbuild.result == 'success' }}

.gitignore

@@ -5,6 +5,7 @@
 .cache/
 hpc/
 QC/
+Deconvolve/
 Demultiplex/
 Genome/
 Align/

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "harpy"
-version = "1.1.1"
+version = "0.0.0"
 authors = [
     {name = "Pavel Dimens"}
 ]

src/harpy/__main__.py

@@ -1,20 +1,6 @@
 #!/usr/bin/env python3
 
-import os
-import sys
-
-try:
-    harpypath = '{CONDA_PREFIX}'.format(**os.environ) + "/bin"
-except:
-    enverror = "\033[1;33mERROR:\033[00m Harpy expects to run from within an active conda environment, but one was not detected."
-    print(enverror, file = sys.stderr)
-    fix = "\033[1;34mSOLUTION:\033[00m Activate the conda environment Harpy was installed into and run Harpy again."
-    print("\n" + fix, file = sys.stderr)
-    print(f"\n\033[1mDetails:\033[00m")
-    details = "In order to work correctly, Harpy expects several software packages to be available in the PATH, which are provided automatically with Harpy's conda-based installation. If you're seeing this message, no active conda environment was detected upon execution, and Harpy exited as an early failsafe against unexpected runtime errors associated with \"missing\" files and packages."
-    print(details, file = sys.stderr)
-    sys.exit(1)
-
+import rich_click as click
 from . import align
 from . import demultiplex
 from . import impute
@@ -27,9 +13,9 @@
 from . import container
 from . import hpc
 from . import resume
+from . import deconvolve
 from .popgroups import popgroup
 from .stitchparams import stitchparams
-import rich_click as click
 
 click.rich_click.USE_MARKDOWN = True
 click.rich_click.SHOW_ARGUMENTS = False
@@ -38,10 +24,13 @@
 click.rich_click.MAX_WIDTH = 75
 click.rich_click.REQUIRED_SHORT_STRING = ""
 click.rich_click.ERRORS_SUGGESTION = "Try the '--help' flag for more information."
-click.rich_click.ERRORS_EPILOGUE = "See the documentation: [link=https://pdimens.github.io/harpy/]https://pdimens.github.io/harpy/[/link]"
+click.rich_click.ERRORS_EPILOGUE = """
+See the documentation: [link=https://pdimens.github.io/harpy/]https://pdimens.github.io/harpy/[/link]
+"""
 
 @click.group(options_metavar='', context_settings={"help_option_names" : ["-h", "--help"]})
-@click.version_option("1.1.1", prog_name="Harpy")
+@click.version_option("0.0.0", prog_name="harpy")
+
 def cli():
     """
     An automated workflow for haplotagging linked-read data
@@ -69,6 +58,7 @@ def cli():
 cli.add_command(container.containerize)
 cli.add_command(hpc.hpc)
 cli.add_command(resume.resume)
+cli.add_command(deconvolve.deconvolve)
 
 ## the modules ##
 click.rich_click.COMMAND_GROUPS = {
@@ -80,9 +70,10 @@ def cli():
             },
             {
                 "name": "Other Commands",
-                "commands": ["resume", "hpc", "preflight", "popgroup", "stitchparams"]
+                "commands": ["resume", "hpc", "preflight", "deconvolve", "popgroup", "stitchparams"]
             }
         ],
  } | simulate.commandstring | hpc.docstring
 
-click.rich_click.OPTION_GROUPS = demultiplex.docstring | preflight.docstring | qc.docstring | align.docstring | snp.docstring | sv.docstring | impute.docstring | phase.docstring | simulate.docstring
+for i in [align, deconvolve, demultiplex, impute, phase, preflight, qc, simulate, snp, sv]:
+    click.rich_click.OPTION_GROUPS |= i.docstring