Merge pull request #117 from CCBR/iss-113

Improve runtest defaults, add changelog, update --version flag, add --force flag, refactor Rlibs
CCBR · Feb 21, 2024 · 71fd514 · 71fd514
2 parents f41e914 + 1979362
commit 71fd514
Show file tree

Hide file tree

Showing 40 changed files with 86,979 additions and 251 deletions.
diff --git a/.github/workflows/lintr.yaml b/.github/workflows/lintr.yaml
@@ -27,4 +27,4 @@ jobs:
         touch hg38.fa genes.gtf hg38.bed hg38.tss.bed hg38_refseq.ucsc Ecoli_GCF_000005845.2_ASM584v2_genomic.fna adapters.fa && \
         snakemake --lint -s /opt2/workflow/Snakefile \
         -d /opt2/output_carlisle --configfile /opt2/.test/config_lint.yaml || \
-        echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'"
+        echo 'There may have been a few warnings or errors. Please read through the log to determine if its harmless.'"
diff --git a/.test/53_H4K20m3_1.R1.fastq.gz b/.test/53_H4K20m3_1.R1.fastq.gz
diff --git a/.test/53_H4K20m3_1.R2.fastq.gz b/.test/53_H4K20m3_1.R2.fastq.gz
diff --git a/.test/53_H4K20m3_2.R1.fastq.gz b/.test/53_H4K20m3_2.R1.fastq.gz
diff --git a/.test/53_H4K20m3_2.R2.fastq.gz b/.test/53_H4K20m3_2.R2.fastq.gz
diff --git a/.test/HN6_H3K4me3_2.R1.fastq.gz b/.test/HN6_H3K4me3_2.R1.fastq.gz
diff --git a/.test/HN6_H3K4me3_2.R2.fastq.gz b/.test/HN6_H3K4me3_2.R2.fastq.gz
diff --git a/.test/HN6_H4K20me3_1.R1.fastq.gz b/.test/HN6_H4K20me3_1.R1.fastq.gz
diff --git a/.test/HN6_H4K20me3_1.R2.fastq.gz b/.test/HN6_H4K20me3_1.R2.fastq.gz
diff --git a/.test/HN6_H4K20me3_2.R1.fastq.gz b/.test/HN6_H4K20me3_2.R1.fastq.gz
diff --git a/.test/HN6_H4K20me3_2.R2.fastq.gz b/.test/HN6_H4K20me3_2.R2.fastq.gz
diff --git a/.test/README.md b/.test/README.md
@@ -1,30 +1,35 @@
-# 12/16/22
+################################################
+# CHANGELOG
+### samples retired 2/6/24
+<!-- VS589.R1.fastq.gz,HN6_H3K4me3_2.R1.fastq.gz
+VS589.R2.fastq.gz,HN6_H3K4me3_2.R2.fastq.gz
+VS594.R1.fastq.gz,HN6_H4K20me3_1.R1.fastq.gz
+VS594.R2.fastq.gz,HN6_H4K20me3_1.R2.fastq.gz
+VS595.R1.fastq.gz,HN6_H4K20me3_2.R1.fastq.gz
+VS595.R2.fastq.gz,HN6_H4K20me3_2.R2.fastq.gz
+VS597.R1.fastq.gz,53_H4K20m3_1.R1.fastq.gz
+VS597.R2.fastq.gz,53_H4K20m3_1.R2.fastq.gz
+VS598.R1.fastq.gz,53_H4K20m3_2.R1.fastq.gz -->
+### 12/16/22
 New test data is added to pipeline
 
+################################################
+# HG38
 # sample location
 /data/CCBR/projects/ccbr1155/CS031014/fastq
 
 # sample ids mapping; original to renamed
 VS586.R1.fastq.gz,HN6_IgG_rabbit_negative_control_1.R1.fastq.gz
 VS586.R2.fastq.gz,HN6_IgG_rabbit_negative_control_1.R2.fastq.gz
-VS588.R1.fastq.gz,HN6_H3K4me3_1.R1.fastq.gz
-VS588.R2.fastq.gz,HN6_H3K4me3_1.R2.fastq.gz
-VS589.R1.fastq.gz,HN6_H3K4me3_2.R1.fastq.gz
-VS589.R2.fastq.gz,HN6_H3K4me3_2.R2.fastq.gz
 VS591.R1.fastq.gz,53_H3K4me3_1.R1.fastq.gz
 VS591.R2.fastq.gz,53_H3K4me3_1.R2.fastq.gz
 VS592.R1.fastq.gz,53_H3K4me3_2.R1.fastq.gz
 VS592.R2.fastq.gz,53_H3K4me3_2.R2.fastq.gz
-VS594.R1.fastq.gz,HN6_H4K20me3_1.R1.fastq.gz
-VS594.R2.fastq.gz,HN6_H4K20me3_1.R2.fastq.gz
-VS595.R1.fastq.gz,HN6_H4K20me3_2.R1.fastq.gz
-VS595.R2.fastq.gz,HN6_H4K20me3_2.R2.fastq.gz
-VS597.R1.fastq.gz,53_H4K20m3_1.R1.fastq.gz
-VS597.R2.fastq.gz,53_H4K20m3_1.R2.fastq.gz
-VS598.R1.fastq.gz,53_H4K20m3_2.R1.fastq.gz
+VS588.R1.fastq.gz,HN6_H3K4me3_1.R1.fastq.gz
+VS588.R2.fastq.gz,HN6_H3K4me3_1.R2.fastq.gz
 
 # subset each of the samples 
 fq_original_loc="/data/CCBR/projects/ccbr1155/CS031014/fastq";\
 test_loc="/data/CCBR_Pipeliner/Pipelines/CARLISLE_dev/.test";\
 fq_list=("HN6_IgG_rabbit_negative_control_1.R1.fastq.gz" "HN6_IgG_rabbit_negative_control_1.R2.fastq.gz" "HN6_H3K4me3_1.R1.fastq.gz" "HN6_H3K4me3_1.R2.fastq.gz" "HN6_H3K4me3_2.R1.fastq.gz" "HN6_H3K4me3_2.R2.fastq.gz" "53_H3K4me3_1.R1.fastq.gz" "53_H3K4me3_1.R2.fastq.gz" "53_H3K4me3_2.R1.fastq.gz" "53_H3K4me3_2.R2.fastq.gz" "HN6_H4K20me3_1.R1.fastq.gz" "HN6_H4K20me3_1.R2.fastq.gz" "HN6_H4K20me3_2.R1.fastq.gz" "HN6_H4K20me3_2.R2.fastq.gz" "53_H4K20m3_1.R1.fastq.gz" "53_H4K20m3_1.R2.fastq.gz" "53_H4K20m3_2.R1.fastq.gz" "53_H4K20m3_2.R2.fastq.gz");\
-for fq in ${fq_list[@]}; do echo $fq; remove_ext=`echo $fq | sed "s/.gz//g"`; zcat $fq_original_loc/$fq | head -1000000 > $test_loc/$remove_ext; gzip $test_loc/$remove_ext; done
+for fq in ${fq_list[@]}; do echo $fq; remove_ext=`echo $fq | sed "s/.gz//g"`; zcat $fq_original_loc/$fq | head -1000000 > $test_loc/$remove_ext; gzip $test_loc/$remove_ext; done
diff --git a/.test/config_lint.yaml b/.test/config_lint.yaml
@@ -145,3 +145,9 @@ spikein_reference:
     fa: "PIPELINE_HOME/resources/spikein/S_cer_S288C_R64.fna"
 
 adapters: "/opt2/output_carlisle/annotation/adapters.fa"
+
+#####################################################################################
+# R Packages
+#####################################################################################
+Rlib_dir: "/data/CCBR_Pipeliner/db/PipeDB/Rlibrary_4.3_carlisle/"
+Rpkg_config: "/opt2/output_carlisle/config//Rpack.config"
diff --git a/.test/contrasts.test.tsv b/.test/contrasts.test.tsv
@@ -1,3 +1,2 @@
 condition1	condition2
-53_H3K4me3	HN6_H3K4me3
-53_H4K20m3	HN6_H4K20me3
+53_H3K4me3	HN6_H3K4me3
diff --git a/.test/contrasts.tinytest.tsv b/.test/contrasts.tinytest.tsv
diff --git a/.test/samples.test.tsv b/.test/samples.test.tsv
@@ -2,9 +2,4 @@ sampleName	replicateNumber	isControl	controlName	controlReplicateNumber	path_to_
 53_H3K4me3	1	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/53_H3K4me3_1.R1.fastq.gz	PIPELINE_HOME/.test/53_H3K4me3_1.R2.fastq.gz
 53_H3K4me3	2	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/53_H3K4me3_2.R1.fastq.gz	PIPELINE_HOME/.test/53_H3K4me3_2.R2.fastq.gz
 HN6_H3K4me3	1	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/HN6_H3K4me3_1.R1.fastq.gz	PIPELINE_HOME/.test/HN6_H3K4me3_1.R2.fastq.gz
-HN6_H3K4me3	2	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/HN6_H3K4me3_2.R1.fastq.gz	PIPELINE_HOME/.test/HN6_H3K4me3_2.R2.fastq.gz
-53_H4K20m3	1	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/53_H4K20m3_1.R1.fastq.gz	PIPELINE_HOME/.test/53_H4K20m3_1.R2.fastq.gz
-53_H4K20m3	2	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/53_H4K20m3_2.R1.fastq.gz	PIPELINE_HOME/.test/53_H4K20m3_2.R2.fastq.gz
-HN6_H4K20me3	1	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/HN6_H4K20me3_1.R1.fastq.gz	PIPELINE_HOME/.test/HN6_H4K20me3_1.R2.fastq.gz
-HN6_H4K20me3	2	N	HN6_IgG_rabbit_negative_control	1	PIPELINE_HOME/.test/HN6_H4K20me3_2.R1.fastq.gz	PIPELINE_HOME/.test/HN6_H4K20me3_2.R2.fastq.gz
 HN6_IgG_rabbit_negative_control	1	Y	-	-	PIPELINE_HOME/.test/HN6_IgG_rabbit_negative_control_1.R1.fastq.gz	PIPELINE_HOME/.test/HN6_IgG_rabbit_negative_control_1.R2.fastq.gz
diff --git a/.test/samples.test_lintr.tsv b/.test/samples.test_lintr.tsv
@@ -2,9 +2,4 @@ sampleName	replicateNumber	isControl	controlName	controlReplicateNumber	path_to_
 53_H3K4me3	1	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/53_H3K4me3_1.R1.fastq.gz	/opt2/.test/53_H3K4me3_1.R2.fastq.gz
 53_H3K4me3	2	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/53_H3K4me3_2.R1.fastq.gz	/opt2/.test/53_H3K4me3_2.R2.fastq.gz
 HN6_H3K4me3	1	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/HN6_H3K4me3_1.R1.fastq.gz	/opt2/.test/HN6_H3K4me3_1.R2.fastq.gz
-HN6_H3K4me3	2	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/HN6_H3K4me3_2.R1.fastq.gz	/opt2/.test/HN6_H3K4me3_2.R2.fastq.gz
-53_H4K20m3	1	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/53_H4K20m3_1.R1.fastq.gz	/opt2/.test/53_H4K20m3_1.R2.fastq.gz
-53_H4K20m3	2	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/53_H4K20m3_2.R1.fastq.gz	/opt2/.test/53_H4K20m3_2.R2.fastq.gz
-HN6_H4K20me3	1	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/HN6_H4K20me3_1.R1.fastq.gz	/opt2/.test/HN6_H4K20me3_1.R2.fastq.gz
-HN6_H4K20me3	2	N	HN6_IgG_rabbit_negative_control	1	/opt2/.test/HN6_H4K20me3_2.R1.fastq.gz	/opt2/.test/HN6_H4K20me3_2.R2.fastq.gz
 HN6_IgG_rabbit_negative_control	1	Y	-	-	/opt2/.test/HN6_IgG_rabbit_negative_control_1.R1.fastq.gz	/opt2/.test/HN6_IgG_rabbit_negative_control_1.R2.fastq.gz
diff --git a/.test/samples.tinytest.tsv b/.test/samples.tinytest.tsv
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -0,0 +1,65 @@
+## CARLISLE development version
+- Refactors R packages to a common source location (#118, @slsevilla)
+- Adds a --force flag to allow for re-initialization of a workdir (#97, @slsevilla)
+- Fixes error with testrun in DESEQ2 (#113, @slsevilla)
+- Decreases the number of samples being run with testrun, essentially running tinytest as default and removing tinytest as an option (#115, @slsevilla)
+- Reads version from VERSION file instead of github repo link (#96, #112, @slsevilla)
+- Added a CHANGELOG (#116, @slsevilla)
+- Fix: RNA report bug, caused by hard-coding of PC1-3, when only PC1-2 were generated (#104, @slsevilla)
+- Minor documentation improvements. (#100, @kelly-sovacool)
+- Fix: allow printing the version or help message even if singularity is not in the path. (#110, @kelly-sovacool)
+
+## CARLISLE v2.4.1 
+- Add GitHub Action to add issues/PRs to personal project boards by @kelly-sovacool in #95
+- Create install script by @kelly-sovacool in #93
+- feat: use summits bed for homer input; save temporary files; fix deseq2 bug by @slsevilla in #108
+- docs: adding citation and DOI to pipeline by @slsevilla in #107
+- Test a dryrun with GitHub Actions by @kelly-sovacool in #94
+
+## CARLISLE v2.4.0
+- Feature- Merged Yue's fork, adding DEEPTOOLS by @slsevilla in #85
+- Feature- Added tracking features from SPOOK by @slsevilla in #88
+- Feature - Dev test run completed by @slsevilla in #89
+- Bug - Fixed bugs related to Biowulf transition
+
+## CARLISLE v2.1.0
+- enhancement
+- update gopeaks resources
+- change SEACR to run "norm" without spikein controls, "non" with spikein controls
+- update docs for changes; provide extra troubleshooting guidance
+- fix GoEnrich bug for failed plots
+
+## CARLISLE v2.0.1
+- fix error when contrasts set to "N"
+- adjust goenrich resources to be more efficient
+
+## CARLISLE 2.0.0
+- Add a MAPQ filter to samtools (rule align)
+- Add GoPeaks MultiQC module
+- Allow for library normalization to occur during first pass
+- Add --broad-cutoff to MACS2 broad peak calling for MACS2
+- Create a spike in QC report
+- Reorganize file structure to help with qthreshold folder
+- Update variable names of all peak caller
+- Merge rules with input/output/wildcard congruency
+- Convert the "spiked" variable to "norm_method
+- Add name of control used to MACS2 peaks
+- Running extra control:sample comparisons that are not needed
+- improved resource allocation
+- test data originally included 1475 jobs, this version includes 1087 jobs (reduction of 25%) despite including additional features
+- moved ~12% of all jobs to local deployment (within SLURM submission)
+
+## CARLISLE 1.2.0
+- merge increases to resources; update workflow img, contributions
+
+## CARLISLE 1.1.1
+- patch for gz bigbed bug
+
+## CARLISLE 1.1.0
+- add broad-cutoff to macs2 broad peaks param settings
+- add non.stringent and non.relaxed to annotation options
+- merge DESEQ and DESEQ2 rules together
+- identify some files as temp
+
+## CARLISLE 1.0.1
+- contains patch for DESEQ error with non hs1 reference samples
diff --git a/annotation/README.md b/annotation/README.md
@@ -1,6 +1,9 @@
+## mm10 download
+https://bitbucket.org/young_computation/rose/src/master/annotation/mm10_refseq.ucsc
+
 `create_hs1_ucsc.sh` script was used to create the hs1 ucsc file.
 
 #### hg19_refseq.ucsc and hg38_refseq.ucsc source
 /data/khanlab3/gb_omics/processed_DATA/YueHu/cutrun_120822# hg19_refseq.ucsc and hg38_refseq.ucsc source
 
-# info on these resources is outlined in: https://github.com/CCBR/CARLISLE/issues/27
+# info on these resources is outlined in: https://github.com/CCBR/CARLISLE/issues/27