Merge pull request #146 from pdimens/cloudspades

add assembly

.deprecated/align-minimap.smk

@@ -7,7 +7,7 @@ from rich.panel import Panel
 from rich import print as rprint
 
 outdir      = config["output_directory"]
-envdir      = os.getcwd() + "/.harpy_envs"
+envdir      = os.path.join(os.getcwd(), ".harpy_envs")
 genomefile 	= config["inputs"]["genome"]
 fqlist      = config["inputs"]["fastq"]
 extra 		= config.get("extra", "") 

.deprecated/phase.bak.smk

@@ -14,7 +14,7 @@ pruning           = config["prune"]
 molecule_distance = config["molecule_distance"]
 extra             = config.get("extra", "") 
 outdir 			  = config["output_directory"]
-envdir      = os.getcwd() + "/.harpy_envs"
+envdir      = os.path.join(os.getcwd(), ".harpy_envs")
 
 if config["noBX"]:
     fragfile = outdir + "/extractHairs/{sample}.unlinked.frags"

.github/filters.yml

@@ -1,8 +1,8 @@
 common: &common
   - 'harpy/_launch.py'
   - 'harpy/_misc.py'
-  - 'harpy/_printing.py'
   - 'harpy/_parsers.py'
+  - 'harpy/_printing.py'
   - 'harpy/_validations.py'
   - 'harpy/__main__.py'
   - 'resources/harpy.yaml'
@@ -11,6 +11,7 @@ common: &common
   - 'resources/buildforCI.sh'
 container: &container
   - 'harpy/_conda.py'
+  - 'harpy/container.py'
   - 'harpy/containerize.smk'
 preflight: &preflight
   - *common
@@ -149,10 +150,11 @@ simreads: &simreads
   - 'extractReads.cpp'
   - 'harpy/bin/10xtoHaplotag.py'
   - 'harpy/scripts/LRSIM_harpy.pl'
-metassembly: &metassembly
+assembly: &assembly
   - *common
   - *container
-  - 'harpy/metassembly.py'
+  - 'harpy/assembly.py'
+  - 'harpy/snakefiles/assembly.smk'
   - 'harpy/snakefiles/metassembly.smk'
 other: &other
   - 'harpy/stitchparams.py'
@@ -175,5 +177,5 @@ modules:
   - *naibr
   - *simvars
   - *simreads
-  - *metassembly
+  - *assembly
   - *other

.github/workflows/tests.yml

@@ -41,7 +41,7 @@ jobs:
       phase: ${{ steps.filter.outputs.phase }}
       simvars: ${{ steps.filter.outputs.simvars }}
       simreads: ${{ steps.filter.outputs.simreads }}
-      metassembly: ${{ steps.filter.outputs.metassembly }}
+      assembly: ${{ steps.filter.outputs.assembly }}
       other: ${{ steps.filter.outputs.other }}
       container: ${{ steps.filter.outputs.container }}
       modules: ${{ steps.filter.outputs.modules }}
@@ -747,10 +747,10 @@ jobs:
         shell: micromamba-shell {0}
         run: harpy simulate linkedreads --quiet -t 4 -n 2  -l 100 -p 50 test/genome/genome.fasta.gz test/genome/genome2.fasta.gz
 
-  extras:
+  assembly:
     needs: [changes, pkgbuild]
-    if: ${{ needs.changes.outputs.other == 'true' && needs.pkgbuild.result == 'success' }}
-    name: harpy extras
+    if: ${{ needs.changes.outputs.assembly == 'true' && needs.pkgbuild.result == 'success' }}
+    name: metassembly
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -776,23 +776,25 @@ jobs:
           resources/buildforCI.sh
       - name: Clear Space
         run: rm -rf /opt/hostedtoolcache && mkdir -p .snakemake/singularity
-      - name: harpy stitchparams
+      - name: Download Singularity Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: deps-image
+          path: .snakemake/singularity
+      - name: test assembly
         shell: micromamba-shell {0}
-        run: harpy stitchparams -o params.file
-      - name: harpy popgroup
+        run: harpy assembly --quiet -m 4000 test/fastq/sample1.*
+      - name: test metassembly
         shell: micromamba-shell {0}
-        run: harpy popgroup test/fastq
-      - name: harpy hpc
+        run: harpy assembly --metassembly spades --quiet -m 4000 test/fastq/sample1.*
+      - name: test metassembly cloud
         shell: micromamba-shell {0}
-        run: |
-          harpy hpc slurm
-          harpy hpc googlebatch
-          harpy hpc lsf
-          harpy hpc htcondor
-  metassembly:
+        run: harpy assembly --metassembly cloudspades --quiet -m 4000 test/fastq/sample1.*
+
+  extras:
     needs: [changes, pkgbuild]
-    if: ${{ needs.changes.outputs.metassembly == 'true' && needs.pkgbuild.result == 'success' }}
-    name: metassembly
+    if: ${{ needs.changes.outputs.other == 'true' && needs.pkgbuild.result == 'success' }}
+    name: harpy extras
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -818,13 +820,16 @@ jobs:
           resources/buildforCI.sh
       - name: Clear Space
         run: rm -rf /opt/hostedtoolcache && mkdir -p .snakemake/singularity
-      - name: Download Singularity Artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: deps-image
-          path: .snakemake/singularity
-      - name: create a metassembly
+      - name: harpy stitchparams
         shell: micromamba-shell {0}
-        run: harpy metassembly --quiet -m 6000 test/fastq/sample1.*
-
-
+        run: harpy stitchparams -o params.file
+      - name: harpy popgroup
+        shell: micromamba-shell {0}
+        run: harpy popgroup test/fastq
+      - name: harpy hpc
+        shell: micromamba-shell {0}
+        run: |
+          harpy hpc slurm
+          harpy hpc googlebatch
+          harpy hpc lsf
+          harpy hpc htcondor

.gitignore

@@ -15,6 +15,7 @@ Simulate/
 Impute/
 Preflight/
 Phase/
+Assembly/
 Metassembly/
 samples.populations
 *.fasta

harpy/__main__.py

@@ -7,7 +7,7 @@
 from . import container
 from . import hpc
 from . import impute
-from . import metassembly
+from . import assembly
 from . import qc
 from . import phase
 from . import preflight
@@ -56,15 +56,16 @@ def cli():
 cli.add_command(hpc.hpc)
 cli.add_command(resume.resume)
 cli.add_command(deconvolve.deconvolve)
-cli.add_command(metassembly.metassembly)
+#cli.add_command(metassembly.metassembly)
+cli.add_command(assembly.assembly)
 
 ## the workflows ##
 click.rich_click.COMMAND_GROUPS = {
     "harpy":
         [
             {
                 "name": "workflows",
-                "commands": ["demultiplex","qc", "align","snp","sv","impute","phase", "simulate", "metassembly"],
+                "commands": ["demultiplex","qc", "align","snp","sv","impute","phase", "simulate", "assembly"],
             },
             {
                 "name": "Other Commands",
@@ -73,5 +74,5 @@ def cli():
         ],
  } | simulate.commandstring | hpc.docstring
 
-for i in [align, deconvolve, demultiplex, impute, phase, preflight, qc, simulate, snp, sv, metassembly]:
+for i in [align, deconvolve, demultiplex, impute, phase, preflight, qc, simulate, snp, sv, assembly]:
     click.rich_click.OPTION_GROUPS |= i.docstring

harpy/_conda.py

@@ -1,4 +1,4 @@
-"""Module that sets up the conda environments and dependencies for all the Harpy workflows"""
+"""Creates environment recipes for all the Harpy workflows"""
 
 import os
 
@@ -16,24 +16,28 @@ def create_conda_recipes():
             "conda-forge::icu",
             "conda-forge::libzlib",
             "conda-forge::xz"
-            ],
-        "assembly": [
-            "conda-forge::python=3"
+        ],
+        "assembly":[
+            "bioconda::arcs",
+            "bioconda::bwa",
+            "bioconda::cloudspades",
+            "bioconda::links",
+            "bioconda::samtools",
+            "bioconda::tigmint"
         ],
         "metassembly": [
             "bioconda::athena_meta=1.2"
         ],
         "phase" : [
             "bioconda::hapcut2",
             "bioconda::whatshap"
-            ],
+        ],
         "qc" : [
-            "bioconda::falco",
             "bioconda::fastp",
             "bioconda::multiqc=1.22",
             "bioconda::pysam=0.22",
             "bioconda::quickdeconvolution"
-            ],
+        ],
         "r" : [
             "conda-forge::pandoc",
             "conda-forge::r-dt",
@@ -49,7 +53,7 @@ def create_conda_recipes():
             "conda-forge::r-viridislite", 
             "conda-forge::r-xml2",
             "r::r-biocircos"
-            ],
+        ],
         "simulations" : [
             "alienzj::msort",
             "bioconda::dwgsim=1.1.14",
@@ -58,16 +62,19 @@ def create_conda_recipes():
             "bioconda::perl-parse-recdescent",
             "conda-forge::numpy",
             "conda-forge::perl"
-            ],
+        ],
+        "spades": [
+            "conda-forge::python=3"
+        ],
+        "stitch" : [
+            "bioconda::r-stitch=1.6.10"
+        ],
         "variants": [
             "bioconda::bcftools=1.20",
             "bioconda::freebayes=1.3.6",
             "bioconda::leviathan",
             "bioconda::naibr-plus"
-            ],
-        "stitch" : [
-            "bioconda::r-stitch=1.6.10"
-            ]
+        ]
     }
 
     os.makedirs(".harpy_envs", exist_ok = True)
@@ -81,7 +88,7 @@ def create_conda_recipes():
             yml.write("\n  - ".join(deps) + "\n")
 
     # post-deployment scripts
-    with open(".harpy_envs/assembly.post-deploy.sh", "w", encoding="utf-8") as shellscript:
+    with open(".harpy_envs/spades.post-deploy.sh", "w", encoding="utf-8") as shellscript:
         shellscript.write("wget -O .spades.tar.gz https://github.com/ablab/spades/releases/download/v4.0.0/SPAdes-4.0.0-Linux.tar.gz\n")
         shellscript.write("tar -xvzf .spades.tar.gz && rm .spades.tar.gz\n")
         shellscript.write("mv SPAdes-4.0.0-Linux/bin/* ${CONDA_PREFIX}/bin && mv SPAdes-4.0.0-Linux/share/* ${CONDA_PREFIX}/share\n")

harpy/_launch.py

@@ -28,7 +28,7 @@ def purge_empty_logs(target_dir):
         if os.path.isdir(logfile) and not os.listdir(logfile):
             os.rmdir(logfile)
 
-def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet):
+def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet, summaryfile = None):
     """launch snakemake with the given commands"""
     if not quiet:
         print_onstart(starttext, workflow.replace("_", " "))
@@ -56,22 +56,22 @@ def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet):
             if process.poll() or iserror(output):
                 exitcode = EXIT_CODE_SUCCESS if process.poll() == 0 else EXIT_CODE_GENERIC_ERROR
                 break
-            while not output.startswith("Job stats:"):
+            while not output.startswith("Job stats:") and not exitcode:
                 # print dependency text only once
                 if "Downloading and installing remote packages" in output or "Running post-deploy" in output:
                     deps = True
-                    deploy_text = "[dim]Installing software dependencies"
+                    deploy_text = "[dim]Installing workflow dependencies"
                     break
                 if "Pulling singularity image" in output:
                     deps = True
-                    deploy_text = "[dim]Downloading software container"
+                    deploy_text = "[dim]Downloading harpy container"
                     break
                 if "Nothing to be" in output:
                     exitcode = EXIT_CODE_SUCCESS
-                    break
                 if "MissingInput" in output:
                     exitcode = EXIT_CODE_GENERIC_ERROR
-                    break
+                if "AmbiguousRuleException" in output or "Error" in output or "Exception" in output:
+                    exitcode = EXIT_CODE_RUNTIME_ERROR
                 output = process.stderr.readline()
 
             # if dependency text present, print pulsing progress bar
@@ -153,7 +153,7 @@ def launch_snakemake(sm_args, workflow, starttext, outdir, sm_logfile, quiet):
             gzip_file(sm_logfile)
             purge_empty_logs(outdir)
             if not quiet:
-                print_onsuccess(outdir)
+                print_onsuccess(outdir, summaryfile)
             sys.exit(0)
         else:
             if exitcode in (1,2):