add logging to outdir

src/harpy/snakefiles/align_bwa.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import sys
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -17,6 +19,10 @@ genome_zip  = True if bn.lower().endswith(".gz") else False
 bn_idx      = f"{bn}.gzi" if genome_zip else f"{bn}.fai"
 skipreports = config["skip_reports"]
 windowsize  = config["depth_windowsize"]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/align_ema.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import shutil
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich.panel import Panel
 from rich import print as rprint
@@ -21,7 +23,11 @@ bn_idx      = f"{bn}.gzi" if genome_zip else f"{bn}.fai"
 envdir      = os.getcwd() + "/.harpy_envs"
 windowsize  = config["depth_windowsize"]
 skipreports = config["skip_reports"]
-
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
+
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"
 

src/harpy/snakefiles/align_strobealign.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import sys
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -19,6 +21,10 @@ windowsize  = config["depth_windowsize"]
 molecule_distance = config["molecule_distance"]
 readlen = config["average_read_length"]
 autolen = isinstance(readlen, str)
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/deconvolve.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import sys
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -13,6 +15,11 @@ kmer_length = config["kmer_length"]
 window_size = config["window_size"]
 density 	= config["density"] 
 dropout     = config["dropout"]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
+
 
 bn_r = r"([_\.][12]|[_\.][FR]|[_\.]R[12](?:\_00[0-9])*)?\.((fastq|fq)(\.gz)?)$"
 samplenames = {re.sub(bn_r, "", os.path.basename(i), flags = re.IGNORECASE) for i in fqlist}

src/harpy/snakefiles/demultiplex_gen1.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import sys
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -13,7 +15,11 @@ I2 = config["inputs"]["I2"]
 samplefile = config["inputs"]["demultiplex_schema"]
 skipreports = config["skip_reports"]
 outdir = config["output_directory"]
-envdir      = os.getcwd() + "/.harpy_envs"
+envdir = os.getcwd() + "/.harpy_envs"
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 def barcodedict(smpl):
     d = {}

src/harpy/snakefiles/impute.smk

@@ -6,6 +6,8 @@ import sys
 import subprocess
 import pandas as pd
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 from snakemake.utils import Paramspace
@@ -18,6 +20,10 @@ skipreports = config["skip_reports"]
 outdir      = config["output_directory"]
 envdir      = os.getcwd() + "/.harpy_envs"
 paramspace  = Paramspace(pd.read_csv(paramfile, sep=r"\s+", skip_blank_lines=True).rename(columns=str.lower), param_sep = "", filename_params = ["k", "s", "ngen", "bxlimit"])
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 with open(biallelic, "r") as f_open:
     contigs = [i.rstrip() for i in f_open.readlines()]

src/harpy/snakefiles/phase.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import sys
 import subprocess
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -19,6 +21,10 @@ skipreports       = config["skip_reports"]
 samples_from_vcf  = config["samples_from_vcf"]
 variantfile       = config["inputs"]["variantfile"]
 bamlist     = config["inputs"]["alignments"]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 # toggle linked-read aware mode
 if config["ignore_bx"]:

src/harpy/snakefiles/preflight_bam.smk

@@ -4,6 +4,8 @@ import os
 import re
 import sys
 import glob
+import logging as pylogging
+from datetime import datetime
 import multiprocessing
 from pathlib import Path
 from rich import print as rprint
@@ -13,6 +15,10 @@ out_dir = config["output_directory"]
 envdir  = os.getcwd() + "/.harpy_envs"
 bamlist = config["inputs"]
 samplenames = {Path(i).stem for i in bamlist}  
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/preflight_fastq.smk

@@ -4,6 +4,8 @@ import os
 import re
 import sys
 import glob
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -13,6 +15,10 @@ out_dir = config["output_directory"]
 envdir      = os.getcwd() + "/.harpy_envs"
 bn_r = r"([_\.][12]|[_\.][FR]|[_\.]R[12](?:\_00[0-9])*)?\.((fastq|fq)(\.gz)?)$"
 samplenames = {re.sub(bn_r, "", os.path.basename(i), flags = re.IGNORECASE) for i in fqlist}
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/qc.smk

@@ -3,6 +3,10 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import sys
+import logging as pylogging
+from datetime import datetime
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -21,6 +25,10 @@ if deconvolve:
     decon_d = deconvolve["density"]
     decon_a = deconvolve["dropout"]
 skipreports  = config["skip_reports"]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 bn_r = r"([_\.][12]|[_\.][FR]|[_\.]R[12](?:\_00[0-9])*)?\.((fastq|fq)(\.gz)?)$"
 samplenames = {re.sub(bn_r, "", os.path.basename(i), flags = re.IGNORECASE) for i in fqlist}

src/harpy/snakefiles/simulate_linkedreads.smk

@@ -4,21 +4,26 @@ import os
 import sys
 import gzip
 import shutil
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
-
 from rich.panel import Panel
 from rich import print as rprint
 
-outdir = config["output_directory"]
+outdir   = config["output_directory"]
 gen_hap1 = config["inputs"]["genome_hap1"]
 gen_hap2 = config["inputs"]["genome_hap2"]
-envdir      = os.getcwd() + "/.harpy_envs"
-
+envdir   = os.getcwd() + "/.harpy_envs"
 barcodes = config.get("barcodes", None)
 if barcodes:
     barcodefile = barcodes
 else:
     barcodefile = f"{outdir}/workflow/input/4M-with-alts-february-2016.txt"
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
+
 
 onsuccess:
     shutil.rmtree('./_Inline', ignore_errors=True)

src/harpy/snakefiles/simulate_snpindel.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import sys
 import os
 import random
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -13,6 +15,11 @@ snp_vcf = config["inputs"].get("snp_vcf", None)
 indel_vcf = config["inputs"].get("indel_vcf", None)
 heterozygosity = config["heterozygosity"]
 outprefix = config["prefix"]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
+
 in_vcfs = []
 snp = False 
 indel = False

src/harpy/snakefiles/simulate_variants.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import sys
 import os
 import random
+import logging as pylogging
+from datetime import datetime
 from rich.panel import Panel
 from rich import print as rprint
 
@@ -14,6 +16,10 @@ genome = config["inputs"]["genome"]
 vcf = config["inputs"].get("vcf", None)
 heterozygosity = float(config["heterozygosity"])
 vcf_correct = "None"
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 if vcf:
     vcf_correct = vcf[:-4] + ".vcf.gz" if vcf.lower().endswith("bcf") else vcf

src/harpy/snakefiles/snp_freebayes.smk

@@ -4,6 +4,8 @@ import os
 import sys
 import gzip
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -40,6 +42,10 @@ else:
             cont,startpos,endpos = line.split()
             intervals.add(f"{cont}:{startpos}-{endpos}")
     regions = dict(zip(intervals, intervals))
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/snp_mpileup.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import sys
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -35,6 +37,10 @@ else:
             cont,startpos,endpos = line.split()
             intervals.add(f"{cont}:{startpos}-{endpos}")
     regions = dict(zip(intervals, intervals))
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/sv_leviathan.smk

@@ -4,6 +4,8 @@ import os
 import re
 import sys
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -22,6 +24,10 @@ bn          = os.path.basename(genomefile)
 genome_zip  = True if bn.lower().endswith(".gz") else False
 if genome_zip:
     bn = bn[:-3]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/sv_leviathan_pop.smk

@@ -3,6 +3,8 @@ containerized: "docker://pdimens/harpy:latest"
 import os
 import re
 import sys
+import logging as pylogging
+from datetime import datetime
 from rich import print as rprint
 from rich.panel import Panel
 
@@ -19,6 +21,10 @@ skipreports = config["skip_reports"]
 bn 			= os.path.basename(genomefile)
 if bn.lower().endswith(".gz"):
     bn = bn[:-3]
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+",

src/harpy/snakefiles/sv_naibr.smk

@@ -4,6 +4,8 @@ import os
 import re
 import sys
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -21,6 +23,10 @@ skipreports = config["skip_reports"]
 bn          = os.path.basename(genomefile)
 genome_zip  = True if bn.lower().endswith(".gz") else False
 bn_idx      = f"{bn}.gzi" if genome_zip else f"{bn}.fai"
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"

src/harpy/snakefiles/sv_naibr_phase.smk

@@ -4,6 +4,8 @@ import os
 import re
 import sys
 import multiprocessing
+import logging as pylogging
+from datetime import datetime
 from pathlib import Path
 from rich import print as rprint
 from rich.panel import Panel
@@ -29,6 +31,10 @@ if vcffile.lower().endswith("bcf"):
     vcfindex = vcffile + ".csi"
 else:
     vcfindex = vcffile + ".tbi"
+os.makedirs(f"{outdir}/logs/snakemake", exist_ok = True)
+dt_string = datetime.now().strftime("%d_%m_%Y-%H_%M_%S")
+extra_logfile_handler = pylogging.FileHandler(f"{outdir}/logs/snakemake/{dt_string}.snakelog")
+logger.logger.addHandler(extra_logfile_handler)
 
 wildcard_constraints:
     sample = "[a-zA-Z0-9._-]+"