Merge pull request #126 from pdimens/progressbar_advanced

advanced progressbar
pdimens · Aug 13, 2024 · 012324f · 012324f
2 parents c0117e6 + 02be8cd
commit 012324f
Show file tree

Hide file tree

Showing 40 changed files with 366 additions and 788 deletions.
diff --git a/.deprecated/align-minimap.smk b/.deprecated/align-minimap.smk
@@ -54,7 +54,7 @@ def get_fq(wildcards):
     r = re.compile(fr".*/({re.escape(wildcards.sample)}){bn_r}", flags = re.IGNORECASE)
     return sorted(list(filter(r.match, fqlist))[:2])
 
-rule genome_setup:
+rule setup_genome:
     input:
         genomefile
     output: 
@@ -73,7 +73,7 @@ rule genome_setup:
         fi
         """
 
-rule genome_faidx:
+rule faidx_genome:
     input: 
         f"Genome/{bn}"
     output: 

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -106,7 +106,7 @@ jobs:
         id: singularity
         shell: micromamba-shell {0}
         if: ${{ needs.changes.outputs.modules == 'true' }}
-        run: harpy qc --skipreports -a test/fastq/sample1.*.fq.gz
+        run: harpy qc --skipreports --quiet test/fastq/sample1.*.fq.gz
       - name: Create Singularity Artifact
         if: ${{ steps.singularity.outcome == 'success' }}
         uses: actions/upload-artifact@v4
@@ -148,7 +148,7 @@ jobs:
           path: .snakemake/singularity
       - name: harpy demultiplex
         shell: micromamba-shell {0}
-        run: harpy demultiplex gen1 --schema test/demux/samples.schema test/demux/Undetermined_S0_L004_R* test/demux/Undetermined_S0_L004_I*
+        run: harpy demultiplex gen1 --quiet --schema test/demux/samples.schema test/demux/Undetermined_S0_L004_R* test/demux/Undetermined_S0_L004_I*
 
   preflight:
     needs: [changes, pkgbuild]
@@ -190,7 +190,7 @@ jobs:
       - name: test preflight bam
         if: always()
         shell: micromamba-shell {0}
-        run: harpy preflight bam test/bam
+        run: harpy preflight bam --quiet test/bam
 
   qc:
     needs: [changes, pkgbuild]
@@ -228,10 +228,10 @@ jobs:
           path: .snakemake/singularity
       - name: harpy qc
         shell: micromamba-shell {0}
-        run: harpy qc -x "--trim_poly_g"   test/fastq
+        run: harpy qc -x "--trim_poly_g" --quiet test/fastq
       - name: harpy qc all options
         shell: micromamba-shell {0}
-        run: harpy qc -a -d -c  test/fastq
+        run: harpy qc -a -d -c --quiet test/fastq
   deconvolve:
     needs: [changes, pkgbuild]
     if: ${{ needs.changes.outputs.deconvolve == 'true' && needs.pkgbuild.result == 'success' }}
@@ -268,7 +268,7 @@ jobs:
           path: .snakemake/singularity
       - name: harpy deconvolve
         shell: micromamba-shell {0}
-        run: harpy deconvolve  test/fastq
+        run: harpy deconvolve --quiet test/fastq
   bwa:
     needs: [changes, pkgbuild]
     if: ${{ needs.changes.outputs.bwa == 'true' && needs.pkgbuild.result == 'success' }}
@@ -305,7 +305,7 @@ jobs:
           path: .snakemake/singularity
       - name: test bwa
         shell: micromamba-shell {0}
-        run: harpy align bwa -g test/genome/genome.fasta.gz  -x "-A 2" test/fastq
+        run: harpy align bwa --quiet -g test/genome/genome.fasta.gz  -x "-A 2" test/fastq
 
   ema:
     needs: [changes, pkgbuild]
@@ -343,7 +343,7 @@ jobs:
           path: .snakemake/singularity
       - name: test ema
         shell: micromamba-shell {0}
-        run: harpy align ema --ema-bins 150 -g test/genome/genome.fasta.gz  -x "-d" test/fastq
+        run: harpy align ema --quiet --ema-bins 150 -g test/genome/genome.fasta.gz -x "-d" test/fastq
 
   strobe:
     needs: [changes, pkgbuild]
@@ -381,7 +381,7 @@ jobs:
           path: .snakemake/singularity
       - name: test strobealign
         shell: micromamba-shell {0}
-        run: harpy align strobe -l 125 -g test/genome/genome.fasta.gz  test/fastq
+        run: harpy align strobe --quiet -l 125 -g test/genome/genome.fasta.gz  test/fastq
 
   mpileup:
     needs: [changes, pkgbuild]
@@ -419,10 +419,10 @@ jobs:
           path: .snakemake/singularity
       - name: snp mpileup
         shell: micromamba-shell {0}
-        run: harpy snp mpileup -r test/positions.bed -g test/genome/genome.fasta.gz -x "--ignore-RG" test/bam
+        run: harpy snp mpileup --quiet -r test/positions.bed -g test/genome/genome.fasta.gz -x "--ignore-RG" test/bam
       - name: snp mpileup-pop
         shell: micromamba-shell {0}
-        run: harpy snp mpileup -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups  test/bam
+        run: harpy snp mpileup --quiet -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups  test/bam
 
   freebayes:
     needs: [changes, pkgbuild]
@@ -460,10 +460,10 @@ jobs:
           path: .snakemake/singularity
       - name: snp freebayes
         shell: micromamba-shell {0}
-        run: harpy snp freebayes -r test/positions.bed -g test/genome/genome.fasta.gz -x "-g 200" test/bam
+        run: harpy snp freebayes --quiet -r test/positions.bed -g test/genome/genome.fasta.gz -x "-g 200" test/bam
       - name: snp freebayes-pop
         shell: micromamba-shell {0}
-        run: harpy snp freebayes -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups  test/bam
+        run: harpy snp freebayes --quiet -r test/positions.bed -o SNP/poptest -g test/genome/genome.fasta.gz -p test/samples.groups  test/bam
 
   impute:
     needs: [changes, pkgbuild]
@@ -501,11 +501,11 @@ jobs:
           path: .snakemake/singularity
       - name: impute
         shell: micromamba-shell {0}
-        run: harpy impute --vcf test/vcf/test.bcf -p test/stitch.params  test/bam
+        run: harpy impute --quiet --vcf test/vcf/test.bcf -p test/stitch.params  test/bam
       - name: impute from vcf
         shell: micromamba-shell {0}
         if: always()
-        run: harpy impute --vcf-samples -o vcfImpute --vcf test/vcf/test.bcf -p test/stitch.params  test/bam
+        run: harpy impute --quiet --vcf-samples -o vcfImpute --vcf test/vcf/test.bcf -p test/stitch.params  test/bam
 
   phase:
     needs: [changes, pkgbuild]
@@ -543,17 +543,17 @@ jobs:
           path: .snakemake/singularity
       - name: phase
         shell: micromamba-shell {0}
-        run: harpy phase --vcf test/vcf/test.bcf  -x "--max_iter 10001" test/bam
+        run: harpy phase --quiet --vcf test/vcf/test.bcf  -x "--max_iter 10001" test/bam
       - name: phase with indels
         shell: micromamba-shell {0}
         if: always()
-        run: harpy phase --vcf test/vcf/test.bcf -o phaseindel -g test/genome/genome.fasta.gz  test/bam
+        run: harpy phase --quiet --vcf test/vcf/test.bcf -o phaseindel -g test/genome/genome.fasta.gz  test/bam
       - name: phase from vcf
         shell: micromamba-shell {0}
         if: always()
         run: |
           cp test/bam/sample1.bam test/bam/pineapple.bam && rename_bam -d test/bam/pineapple.bam pineapple1
-          harpy phase --vcf-samples -o phasevcf --vcf test/vcf/test.bcf  test/bam
+          harpy phase --quiet --vcf-samples -o phasevcf --vcf test/vcf/test.bcf  test/bam
 
   leviathan:
     needs: [changes, pkgbuild]
@@ -591,12 +591,12 @@ jobs:
           path: .snakemake/singularity
       - name: leviathan
         shell: micromamba-shell {0}
-        run: harpy sv leviathan -s 100 -b 1 -g test/genome/genome.fasta.gz  -x "-M 2002" test/bam
+        run: harpy sv leviathan --quiet -s 100 -b 1 -g test/genome/genome.fasta.gz  -x "-M 2002" test/bam
         continue-on-error: true
       - name: leviathan-pop
         if: always()
         shell: micromamba-shell {0}
-        run: harpy sv leviathan -s 100 -b 1 -g test/genome/genome.fasta.gz -o SV/leviathanpop -p test/samples.groups  test/bam
+        run: harpy sv leviathan --quiet -s 100 -b 1 -g test/genome/genome.fasta.gz -o SV/leviathanpop -p test/samples.groups  test/bam
 
   naibr:
     needs: [changes, pkgbuild]
@@ -634,20 +634,20 @@ jobs:
           path: .snakemake/singularity
       - name: naibr
         shell: micromamba-shell {0}
-        run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/naibr  -x "-min_sv 5000" test/bam_phased && rm -r Genome
+        run: harpy sv naibr --quiet -g test/genome/genome.fasta.gz -o SV/naibr  -x "-min_sv 5000" test/bam_phased && rm -r Genome
       - name: naibr pop
         if: always()
         shell: micromamba-shell {0}
-        run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/pop -p test/samples.groups  test/bam_phased && rm -r Genome
+        run: harpy sv naibr --quiet -g test/genome/genome.fasta.gz -o SV/pop -p test/samples.groups  test/bam_phased && rm -r Genome
       - name: naibr with phasing
         if: always()
         shell: micromamba-shell {0}
         run: |
-          harpy sv naibr -g test/genome/genome.fasta.gz -o SV/phase -v test/vcf/test.phased.bcf  test/bam && rm -r Genome
+          harpy sv naibr --quiet -g test/genome/genome.fasta.gz -o SV/phase -v test/vcf/test.phased.bcf  test/bam && rm -r Genome
       - name: naibr pop with phasing
         if: always()
         shell: micromamba-shell {0}
-        run: harpy sv naibr -g test/genome/genome.fasta.gz -o SV/phasepop -v test/vcf/test.phased.bcf -p test/samples.groups  test/bam && rm -r Genome
+        run: harpy sv naibr --quiet -g test/genome/genome.fasta.gz -o SV/phasepop -v test/vcf/test.phased.bcf -p test/samples.groups  test/bam && rm -r Genome
 
 
   simulate_variants:
@@ -687,26 +687,26 @@ jobs:
       - name: simulate random snps/indels
         shell: micromamba-shell {0}
         run: |
-          harpy simulate snpindel --snp-count 10 --indel-count 10 -z 0.5  test/genome/genome.fasta.gz
-          harpy simulate snpindel --prefix Simulate/snpvcf --snp-vcf Simulate/snpindel/sim.snpindel.snp.hap1.vcf --indel-vcf Simulate/snpindel/sim.snpindel.indel.hap1.vcf  test/genome/genome.fasta.gz
+          harpy simulate snpindel --quiet --snp-count 10 --indel-count 10 -z 0.5  test/genome/genome.fasta.gz
+          harpy simulate snpindel --quiet --prefix Simulate/snpvcf --snp-vcf Simulate/snpindel/sim.snpindel.snp.hap1.vcf --indel-vcf Simulate/snpindel/sim.snpindel.indel.hap1.vcf  test/genome/genome.fasta.gz
       - name: simulate inversions
         shell: micromamba-shell {0}
         if: always()
         run: |
-          harpy simulate inversion --count 10 -z 0.5 test/genome/genome.fasta.gz
-          harpy simulate inversion --prefix Simulate/invvcf --vcf Simulate/inversion/sim.inversion.hap1.vcf  test/genome/genome.fasta.gz
+          harpy simulate inversion --quiet --count 10 -z 0.5 test/genome/genome.fasta.gz
+          harpy simulate inversion --quiet --prefix Simulate/invvcf --vcf Simulate/inversion/sim.inversion.hap1.vcf  test/genome/genome.fasta.gz
       - name: simulate cnv
         shell: micromamba-shell {0}
         if: always()
         run: |
-          harpy simulate cnv --count 10 -z 0.5 test/genome/genome.fasta.gz
-          harpy simulate cnv --prefix Simulate/cnvvcf --vcf Simulate/cnv/sim.cnv.hap1.vcf  test/genome/genome.fasta.gz
+          harpy simulate cnv --quiet --count 10 -z 0.5 test/genome/genome.fasta.gz
+          harpy simulate cnv --quiet --prefix Simulate/cnvvcf --vcf Simulate/cnv/sim.cnv.hap1.vcf  test/genome/genome.fasta.gz
       - name: simulate translocations
         shell: micromamba-shell {0}
         if: always()
         run: |
-          harpy simulate translocation --count 10 -z 0.5 test/genome/genome.fasta.gz
-          harpy simulate translocation --prefix Simulate/transvcf --vcf Simulate/translocation/sim.translocation.hap1.vcf  test/genome/genome.fasta.gz
+          harpy simulate translocation --quiet --count 10 -z 0.5 test/genome/genome.fasta.gz
+          harpy simulate translocation --quiet --prefix Simulate/transvcf --vcf Simulate/translocation/sim.translocation.hap1.vcf  test/genome/genome.fasta.gz
 
   simulate_linkedreads:
     needs: [changes, pkgbuild]
@@ -744,7 +744,7 @@ jobs:
           path: .snakemake/singularity
       - name: simulate linked reads
         shell: micromamba-shell {0}
-        run: harpy simulate linkedreads  -t 4 -n 2  -l 100 -p 50 test/genome/genome.fasta.gz test/genome/genome2.fasta.gz
+        run: harpy simulate linkedreads --quiet -t 4 -n 2  -l 100 -p 50 test/genome/genome.fasta.gz test/genome/genome2.fasta.gz
 
   extras:
     needs: [changes, pkgbuild]

diff --git a/harpy/align.py b/harpy/align.py
@@ -94,8 +94,6 @@ def bwa(inputs, output_dir, genome, depth_window, threads, keep_unmapped, extra_
     command += f"--configfile {workflowdir}/config.yaml "
     if hpc:
         command += f"--workflow-profile {hpc} "
-    if quiet:
-        command += "--quiet all "
     if snakemake is not None:
         command += snakemake
 
@@ -131,7 +129,7 @@ def bwa(inputs, output_dir, genome, depth_window, threads, keep_unmapped, extra_
 
     generate_conda_deps()
     start_text =  f"Samples: {sample_count}\nOutput Directory: {output_dir}\nLog: {sm_log}"
-    launch_snakemake(command, "align_bwa", start_text, output_dir, sm_log)
+    launch_snakemake(command, "align_bwa", start_text, output_dir, sm_log, quiet)
 
 @click.command(no_args_is_help = True, epilog = "See the documentation for more information: https://pdimens.github.io/harpy/modules/align/ema")
 @click.option('-x', '--extra-params', type = str, help = 'Additional ema align parameters, in quotes')
@@ -172,8 +170,6 @@ def ema(inputs, output_dir, platform, whitelist, genome, depth_window, keep_unma
     command += f"--configfile {workflowdir}/config.yaml "
     if hpc:
         command += f"--workflow-profile {hpc} "
-    if quiet:
-        command += "--quiet all "
     if snakemake is not None:
         command += snakemake
 
@@ -225,7 +221,7 @@ def ema(inputs, output_dir, platform, whitelist, genome, depth_window, keep_unma
 
     generate_conda_deps()
     start_text = f"Samples: {sample_count}\nPlatform: {platform}\nOutput Directory: {output_dir}/\nLog: {sm_log}"
-    launch_snakemake(command, "align_ema", start_text, output_dir, sm_log)
+    launch_snakemake(command, "align_ema", start_text, output_dir, sm_log, quiet)
 
 @click.command(no_args_is_help = True, epilog= "See the documentation for more information: https://pdimens.github.io/harpy/modules/align/minimap/")
 @click.option('-g', '--genome', type=click.Path(exists=True, dir_okay=False, readable=True), required = True, help = 'Genome assembly for read mapping')
@@ -266,10 +262,8 @@ def strobe(inputs, output_dir, genome, read_length, keep_unmapped, depth_window,
     command += f"--configfile {workflowdir}/config.yaml "
     if hpc:
         command += f"--workflow-profile {hpc} "
-    if quiet:
-        command += "--quiet all "
     if snakemake is not None:
-        command +=  snakemake
+        command += snakemake
 
     os.makedirs(f"{workflowdir}/", exist_ok= True)
     fqlist, sample_count = parse_fastq_inputs(inputs)
@@ -305,7 +299,7 @@ def strobe(inputs, output_dir, genome, read_length, keep_unmapped, depth_window,
 
     generate_conda_deps()
     start_text =  f"Samples: {sample_count}\nOutput Directory: {output_dir}\nLog: {sm_log}"
-    launch_snakemake(command, "align_strobe", start_text, output_dir, sm_log)
+    launch_snakemake(command, "align_strobe", start_text, output_dir, sm_log, quiet)
 
 align.add_command(bwa)
 align.add_command(ema)

diff --git a/harpy/deconvolve.py b/harpy/deconvolve.py
@@ -51,8 +51,6 @@ def deconvolve(inputs, output_dir, kmer_length, window_size, density, dropout, t
     command += f"--configfile {workflowdir}/config.yaml "
     if hpc:
         command += f"--workflow-profile {hpc} "
-    if quiet:
-        command += "--quiet all "
     if snakemake is not None:
         command += snakemake
 
@@ -79,4 +77,4 @@ def deconvolve(inputs, output_dir, kmer_length, window_size, density, dropout, t
 
     generate_conda_deps()
     start_text =  f"Samples: {sample_count}\nOutput Directory: {output_dir}/\nLog: {sm_log}"
-    launch_snakemake(command, "deconvolve", start_text, output_dir, sm_log)
+    launch_snakemake(command, "deconvolve", start_text, output_dir, sm_log, quiet)
diff --git a/harpy/demultiplex.py b/harpy/demultiplex.py
@@ -64,8 +64,6 @@ def gen1(r1_fq, r2_fq, i1_fq, i2_fq, output_dir, schema, threads, snakemake, ski
     command += f"--configfile {workflowdir}/config.yaml "
     if hpc:
         command += f"--workflow-profile {hpc} "
-    if quiet:
-        command += "--quiet all "
     if snakemake is not None:
         command += snakemake
 
@@ -92,6 +90,6 @@ def gen1(r1_fq, r2_fq, i1_fq, i2_fq, output_dir, schema, threads, snakemake, ski
 
     generate_conda_deps()
     start_text = f"Haplotag Type: Generation I\nDemultiplex Schema: {schema}\nOutput Directory: {output_dir}\nLog: {sm_log}"
-    launch_snakemake(command, "demultiplex_gen1", start_text, output_dir, sm_log)
+    launch_snakemake(command, "demultiplex_gen1", start_text, output_dir, sm_log, quiet)
 
 demultiplex.add_command(gen1)