Federica ac clean

.gitignore

@@ -33,7 +33,41 @@ AutoDict_*
 *.tmp
 *.pcm
 *.o
+*.swo
+*.swm
+*.swn
+*Test*
+*TEST*
 
 # Output dirs in signal and bkg
 Signal/outdir_*
+Signal/card*
+Trees2WS/TEST*
+Trees2WS/outdir*
+Plots/SplusBModel*
+Trees2WS/Merge*
+Plots/pkl*
+Trees2WS/card*
+Trees2WS/scrip*
+
+
 Background/outdir_*
+Background/cards*
+Background/bin*
+Background/lib*
+Background/plot*
+
+
+Combine/runFits*
+Combine/runImpact*
+Combine/t2w_jobs*
+Combine/Models*
+Combine/plot*
+
+Datacard/yields*
+Datacard/cards*
+Datacard/Models*
+Merge
+
+
+Combine/Checks/Bias_nominal/*/

Background/RunBackgroundScripts.py

@@ -44,6 +44,8 @@ def leave():
     options['lumi']         = lumiMap[_cfg['year']]
     options['batch']        = _cfg['batch']
     options['queue']        = _cfg['queue']
+    options['xvar']         = _cfg['xvar']
+    options['plotdir']      = _cfg['plotdir'] if 'plotdir' in _cfg else swd__
 
     # Options from command line
     options['mode']                    = opt.mode
@@ -69,7 +71,8 @@ def leave():
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 # If cat == auto: extract list of categories from datafile
 if options['cats'] == 'auto':
-  options['cats'] = extractListOfCatsFromData(options['dataFile'])
+   print options['dataFile']
+   options['cats'] = extractListOfCatsFromData(options['dataFile'])
 options['nCats'] = len(options['cats'].split(","))
 
 # Add dummy entries for procs and signalFitWSFile (used in old plotting script)
@@ -82,6 +85,7 @@ def leave():
 print " --> Categories: %s"%options['cats']
 print " --> Extension: %s"%options['ext']
 print " --> Category offset: %g"%options['catOffset']
+print " --> Variable to fit: %s"%options['xvar']
 print " --> Year: %s ::: Corresponds to intLumi = %s fb^-1"%(options['year'],options['lumi'])
 print ""
 print " --> Job information:"

Background/config_test.py

@@ -2,15 +2,17 @@
 
 backgroundScriptCfg = {
 
-  # Setup
-  'inputWSDir':'/vols/cms/jl2117/hgg/ws/UL/Sept20/merged_data', # location of 'allData.root' file
-  'cats':'auto', # auto: automatically inferred from input ws
-  'catOffset':0, # add offset to category numbers (useful for categories from different allData.root files)  
-  'ext':'test', # extension to add to output directory
-  'year':'combined', # Use combined when merging all years in category (for plots)
+    # Setup
+    'inputWSDir':'cards/', # location of 'allData.root' file
+    'cats':'auto', # auto: automatically inferred from input ws
+    'catOffset':0, # add offset to category numbers (useful for categories from different allData.root files)  
+    'ext':'allData', # extension to add to output directory
+    'year':'combined', # Use combined when merging all years in category (for plots)
+    'xvar': 'CMS_hgg_mass', # not yet used, should be passed to the C++ macros
+    'plotdir': 'plots',
 
-  # Job submission options
-  'batch':'IC', # [condor,SGE,IC,local]
-  'queue':'hep.q' # for condor e.g. microcentury
+    # Job submission options
+    'batch':'condor', # [condor,SGE,IC,Rome,local]
+    'queue':'espresso' # for condor e.g. espresso
 
 }

Background/runBackgroundScripts.sh

@@ -20,6 +20,7 @@ BATCH=""
 QUEUE=""
 YEAR="2016"
 CATOFFSET=0
+PLOTDIR=""
 
 usage(){
 	echo "The script runs background scripts:"
@@ -36,21 +37,22 @@ echo "--pseudoDataOnly) "
 echo "--pseudoDataDat)"
 echo "--sigFile) "
 echo "--bkgPlotsOnly)"
-echo "--seed) for pseudodata random number gen seed (default $SEED)"
+echo "--seed) for pseudodata random number gen seed (default $SEED))"
 echo "--intLumi) specified in fb^-{1} (default $INTLUMI)) "
 echo "--year) dataset year (default $YEAR)) "
 echo "--isData) specified in fb^-{1} (default $DATA)) "
 echo "--unblind) specified in fb^-{1} (default $UNBLIND)) "
 echo "--batch) which batch system to use (None (''),HTCONDOR,IC) (default '$BATCH')) "
 echo "--queue) queue to submit jobs to (specific to batch))"
+echo "--pdir) directory where to put the plots)"
 }
 
 
 #------------------------------ parsing
 
 
 # options may be followed by one colon to indicate they have a required argument
-if ! options=$(getopt -u -o hi:p:f: -l help,inputFile:,procs:,flashggCats:,ext:,catOffset:,fTestOnly,pseudoDataOnly,bkgPlotsOnly,pseudoDataDat:,sigFile:,seed:,intLumi:,year:,unblind,isData,batch:,queue: -- "$@")
+if ! options=$(getopt -u -o hi:p:f: -l help,inputFile:,procs:,flashggCats:,ext:,catOffset:,fTestOnly,pseudoDataOnly,bkgPlotsOnly,pseudoDataDat:,sigFile:,seed:,intLumi:,year:,unblind,isData,batch:,queue:,pdir: -- "$@")
 then
 # something went wrong, getopt will put out an error message for us
 exit 1
@@ -78,6 +80,7 @@ case $1 in
 --unblind) UNBLIND=1;;
 --batch) BATCH=$2; shift;;
 --queue) QUEUE=$2; shift;;
+--pdir) PLOTDIR=$2; shift;;
 
 (--) shift; break;;
 (-*) usage; echo "$0: error - unrecognized option $1" 1>&2; usage >> /dev/stderr; exit 1;;
@@ -88,7 +91,11 @@ done
 
 
 OUTDIR="outdir_${EXT}"
-echo "[INFO] outdir is $OUTDIR, INTLUMI $INTLUMI" 
+if [[ $PLOTDIR == "" ]]; then
+    PLOTDIR=$OUTDIR
+fi
+
+echo "[INFO] outdir is $OUTDIR, plotdir is $PLOTDIR INTLUMI $INTLUMI" 
 
 if [ $ISDATA == 1 ]; then
 DATAEXT="-Data"
@@ -97,6 +104,7 @@ echo "INTLUMI is $INTLUMI, YEAR is $YEAR"
 OUTDIR="outdir_${EXT}"
 
 mkdir -p $OUTDIR
+mkdir -p $PLOTDIR
 
 if [ $FTESTONLY == 0 -a $PSEUDODATAONLY == 0 -a $BKGPLOTSONLY == 0 ]; then
 #IF not particular script specified, run all!
@@ -147,7 +155,7 @@ if [ $FTESTONLY == 1 ]; then
 
 echo "--------------------------------------"
 echo "Running Background F-Test"
-echo "-->Greate background model"
+echo "-->Create background model"
 echo "--------------------------------------"
 if [ $UNBLIND == 1 ]; then
 OPT=" --unblind"
@@ -159,8 +167,14 @@ if [ $ISDATA == 1 ]; then
 OPT=" --isData 1"
 fi
 
+mkdir -p "${PLOTDIR}/bkgfTest${DATAEXT}"
+if test -f "/afs/cern.ch/user/g/gpetrucc/php/index.php"; then
+    cp "/afs/cern.ch/user/g/gpetrucc/php/index.php" "${PLOTDIR}/bkgfTest${DATAEXT}"
+elif test -f "/cmshome/dimarcoe/php/index.php"; then
+    cp "/cmshome/dimarcoe/php/index.php" "${PLOTDIR}/bkgfTest${DATAEXT}"
+fi
 echo " ./bin/fTest -i $FILE --saveMultiPdf $OUTDIR/CMS-HGG_multipdf_$EXT_$CATS.root  -D $OUTDIR/bkgfTest$DATAEXT -f $CATS $OPT --year $YEAR --catOffset $CATOFFSET"
-./bin/fTest -i $FILE --saveMultiPdf $OUTDIR/CMS-HGG_multipdf_$EXT_$CATS.root  -D $OUTDIR/bkgfTest$DATAEXT -f $CATS $OPT --year $YEAR --catOffset $CATOFFSET
+./bin/fTest -i $FILE --saveMultiPdf $OUTDIR/CMS-HGG_multipdf_$EXT_$CATS.root  -D $OUTDIR/bkgfTest$DATAEXT -P $PLOTDIR/bkgfTest$DATAEXT -f $CATS $OPT --year $YEAR --catOffset $CATOFFSET
 
 OPT=""
 fi

Background/run_sequence.sh

@@ -0,0 +1,2 @@
+#python scripts/mergeMultiYearsData.py -i cards/cards_current -o cards/cards_current/data_Run2/allData.root
+python RunBackgroundScripts.py --inputConfig config_test.py --mode fTestParallel

Background/scripts/mergeMultiYearsData.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python                                                                                                                                                                                                                    
+
+import sys
+import glob
+import ROOT
+from optparse import OptionParser
+from commonTools import *
+
+def get_options():
+    parser = OptionParser()
+    parser.add_option("-i","--inputdir", dest="idir", default="cards/cards_current", help="Input directory")
+    parser.add_option("-o","--outfile", dest="outfile", default="allData.root", help="Outputfile")
+    parser.add_option("--years", dest="years", default="2016,2017,2018", help="Merge the datasets of the following years")
+    return parser.parse_args()
+(opt,args) = get_options()
+
+files = args[:]
+
+
+#Extract all files to be merged                                                                                                                                                                                                          
+fNames = {}
+for year in opt.years.split(","): fNames[year] = glob.glob("%s/data_%s/output_Data_13TeV.root"%(opt.idir,year))
+
+cats = extractListOfCats(fNames[opt.years.split(",")[0]]).split(',')
+
+# Define ouput merged workspace
+print " --> Merging output workspaces"
+mergedWS = ROOT.RooWorkspace("cms_hgg_13TeV","cms_hgg_13TeV")
+mergedWS.imp = getattr(mergedWS,"import")
+
+# Extract merged datasets
+data_merged = {}
+data_merged_names = []
+for cat in cats:
+    data_merged["Data_13TeV_%s" % cat] = ROOT.TFile(fNames[opt.years.split(",")[0]][0]).Get("tagsDumper/cms_hgg_13TeV").data("Data_13TeV_%s" % cat).emptyClone("Data_13TeV_%s" % cat)
+    data_merged_names.append( data_merged["Data_13TeV_%s" % cat].GetName() )
+
+for year, fNames in fNames.iteritems():
+    for fName in fNames:
+        for cat in cats:
+            d = ROOT.TFile(fName).Get("tagsDumper/cms_hgg_13TeV").data("Data_13TeV_%s" % cat)
+            print "YEAR = %-6s, CAT = %-30s, n = %d" % (year,cat,d.numEntries())
+            for i in range(d.numEntries()):
+                p = d.get(i)
+                data_merged["Data_13TeV_%s" % cat].add(p)
+
+print " --> Writing to: %s"%(opt.outfile)
+f = ROOT.TFile(opt.outfile,"RECREATE")
+f.mkdir("tagsDumper")
+f.cd("tagsDumper")
+for _data in data_merged.itervalues(): mergedWS.imp(_data)
+mergedWS.Write()
+f.Close()

Background/src/PdfModelBuilder.cc

@@ -17,6 +17,7 @@
 #include "RooConstVar.h"
 #include "RooFitResult.h"
 #include "RooRandom.h"
+#include <stdio.h>
 
 #include "boost/algorithm/string/split.hpp"
 #include "boost/algorithm/string/classification.hpp"
@@ -92,6 +93,10 @@ RooAbsPdf* PdfModelBuilder::getChebychev(string prefix, int order){
 }
 
 RooAbsPdf* PdfModelBuilder::getBernstein(string prefix, int order){
+
+  //if (order >3) {return NULL;}
+
+
 
   RooArgList *coeffList = new RooArgList();
   //coeffList->add(RooConst(1.0)); // no need for cnstant in this interface
@@ -123,9 +128,9 @@ RooAbsPdf* PdfModelBuilder::getBernstein(string prefix, int order){
   } else if (order==6) {
 	RooBernsteinFast<6> *bern = new RooBernsteinFast<6>(prefix.c_str(),prefix.c_str(),*obs_var,*coeffList);
   	return bern;
-//  } else if (order==7) {
-//	RooBernsteinFast<7> *bern = new RooBernsteinFast<7>(prefix.c_str(),prefix.c_str(),*obs_var,*coeffList);
- // 	return bern;
+  } else if (order==7) {
+	RooBernsteinFast<7> *bern = new RooBernsteinFast<7>(prefix.c_str(),prefix.c_str(),*obs_var,*coeffList);
+  	return bern;
   } else {
 	return NULL;
   }

Background/test/fTest.cpp

@@ -64,6 +64,7 @@ RooRealVar *intLumi_ = new RooRealVar("IntLumi","hacked int lumi", 1000.);
 TRandom3 *RandomGen = new TRandom3();
 
 RooAbsPdf* getPdf(PdfModelBuilder &pdfsModel, string type, int order, const char* ext=""){
+
 
   if (type=="Bernstein") return pdfsModel.getBernstein(Form("%s_bern%d",ext,order),order); 
   else if (type=="Chebychev") return pdfsModel.getChebychev(Form("%s_cheb%d",ext,order),order); 
@@ -608,6 +609,7 @@ int main(int argc, char* argv[]){
   int catOffset;
   string datfile;
   string outDir;
+  string plotDir;
   string outfilename;
   bool is2011=false;
   bool verbose=false;
@@ -624,7 +626,8 @@ int main(int argc, char* argv[]){
     ("ncats,c", po::value<int>(&ncats)->default_value(5),                                       "Number of categories")
     ("singleCat", po::value<int>(&singleCategory)->default_value(-1),                           "Run A single Category")
     ("datfile,d", po::value<string>(&datfile)->default_value("dat/fTest.dat"),                  "Right results to datfile for BiasStudy")
-    ("outDir,D", po::value<string>(&outDir)->default_value("plots/fTest"),                      "Out directory for plots")
+    ("outDir,D", po::value<string>(&outDir)->default_value("plots/fTest"),                      "Out directory for results")
+    ("plotDir,P", po::value<string>(&plotDir)->default_value("plots/fTest"),                    "Out directory for plots")
     ("saveMultiPdf", po::value<string>(&outfilename),         					"Save a MultiPdf model with the appropriate pdfs")
     ("runFtestCheckWithToys", 									"When running the F-test, use toys to calculate pvals (and make plots) ")
     ("is2011",                                                                                  "Run 2011 config")
@@ -676,6 +679,7 @@ int main(int argc, char* argv[]){
   }
 
   system(Form("mkdir -p %s",outDir.c_str()));
+  system(Form("mkdir -p %s",plotDir.c_str()));
   TFile *inFile = TFile::Open(fileName.c_str());
   RooWorkspace *inWS;
 	if(isFlashgg_){
@@ -845,14 +849,14 @@ int main(int argc, char* argv[]){
 					if (chi2<0. && order>1) chi2=0.;
 					if (prev_pdf!=NULL){
 						prob = getProbabilityFtest(chi2,order-prev_order,prev_pdf,bkgPdf,mass,data
-								,Form("%s/Ftest_from_%s%d_cat%d.pdf",outDir.c_str(),funcType->c_str(),order,(cat+catOffset)));
+                                                                           ,Form("%s/Ftest_from_%s%d_cat%d.pdf",plotDir.c_str(),funcType->c_str(),order,(cat+catOffset)));
 						std::cout << "[INFO]  F-test Prob(chi2>chi2(data)) == " << prob << std::endl;
 					} else {
 						prob = 0;
 					}
 					double gofProb=0;
 					// otherwise we get it later ...
-					if (!saveMultiPdf) plot(mass,bkgPdf,data,Form("%s/%s%d_cat%d.pdf",outDir.c_str(),funcType->c_str(),order,(cat+catOffset)),flashggCats_,fitStatus,&gofProb);
+					if (!saveMultiPdf) plot(mass,bkgPdf,data,Form("%s/%s%d_cat%d.pdf",plotDir.c_str(),funcType->c_str(),order,(cat+catOffset)),flashggCats_,fitStatus,&gofProb);
 					cout << "[INFO]\t " << *funcType << " " << order << " " << prevNll << " " << thisNll << " " << chi2 << " " << prob << endl;
 					//fprintf(resFile,"%15s && %d && %10.2f && %10.2f && %10.2f \\\\\n",funcType->c_str(),order,thisNll,chi2,prob);
 					prevNll=thisNll;
@@ -908,7 +912,7 @@ int main(int argc, char* argv[]){
 
 						// Calculate goodness of fit for the thing to be included (will use toys for lowstats)!
 						double gofProb =0; 
-						plot(mass,bkgPdf,data,Form("%s/%s%d_cat%d.pdf",outDir.c_str(),funcType->c_str(),order,(cat+catOffset)),flashggCats_,fitStatus,&gofProb);
+						plot(mass,bkgPdf,data,Form("%s/%s%d_cat%d.pdf",plotDir.c_str(),funcType->c_str(),order,(cat+catOffset)),flashggCats_,fitStatus,&gofProb);
 
 						if ((prob < upperEnvThreshold) ) { // Looser requirements for the envelope
 
@@ -944,7 +948,7 @@ int main(int argc, char* argv[]){
 		choices_envelope_vec.push_back(choices_envelope);
 		pdfs_vec.push_back(pdfs);
 
-		plot(mass,pdfs,data,Form("%s/truths_cat%d",outDir.c_str(),(cat+catOffset)),flashggCats_,cat);
+		plot(mass,pdfs,data,Form("%s/truths_cat%d",plotDir.c_str(),(cat+catOffset)),flashggCats_,cat);
 
 		if (saveMultiPdf){
 
@@ -967,6 +971,7 @@ int main(int argc, char* argv[]){
 			//double check the best pdf!
 			int bestFitPdfIndex = getBestFitFunction(pdf,data,&catIndex,!verbose);
 			catIndex.setIndex(bestFitPdfIndex);
+
 			std::cout << "// ------------------------------------------------------------------------- //" <<std::endl; 
 			std::cout << "[INFO] Created MultiPdf " << pdf->GetName() << ", in Category " << cat << " with a total of " << catIndex.numTypes() << " pdfs"<< std::endl;
 			storedPdfs.Print();
@@ -983,7 +988,7 @@ int main(int argc, char* argv[]){
 			outputws->import(catIndex);
 			outputws->import(dataBinned);
 			outputws->import(*data);
-			plot(mass,pdf,&catIndex,data,Form("%s/multipdf_%s",outDir.c_str(),catname.c_str()),flashggCats_,cat,bestFitPdfIndex);
+			plot(mass,pdf,&catIndex,data,Form("%s/multipdf_%s",plotDir.c_str(),catname.c_str()),flashggCats_,cat,bestFitPdfIndex);
 
 		}
 

Background/tools/submissionTools.py

@@ -59,7 +59,7 @@ def writeSubFiles(_opts):
         c = _opts['cats'].split(",")[cidx]
         co = _opts['catOffset']+cidx
         _f.write("if [ $1 -eq %g ]; then\n"%cidx)
-        _cmd = "%s/runBackgroundScripts.sh -i %s -p %s -f %s --ext %s --catOffset %g --intLumi %s --year %s --batch %s --queue %s --sigFile %s --isData --fTest"%(bwd__,_opts['dataFile'],_opts['procs'],c,_opts['ext'],co,_opts['lumi'],_opts['year'],_opts['batch'],_opts['queue'],_opts['signalFitWSFile'])
+        _cmd = "%s/runBackgroundScripts.sh -i %s -p %s -f %s --ext %s --catOffset %g --intLumi %s --year %s --batch %s --queue %s --sigFile %s --isData --fTest --pdir %s"%(bwd__,_opts['dataFile'],_opts['procs'],c,_opts['ext'],co,_opts['lumi'],_opts['year'],_opts['batch'],_opts['queue'],_opts['signalFitWSFile'],_opts['plotdir'])
         _f.write("  %s\n"%_cmd)
         _f.write("fi\n")
 
@@ -73,7 +73,7 @@ def writeSubFiles(_opts):
     _fsub.close()
 
   # SGE...
-  if (_opts['batch'] == "IC")|(_opts['batch'] == "SGE")|(_opts['batch'] == "local" ):
+  if (_opts['batch'] == "IC")|(_opts['batch'] == "SGE")|(_opts['batch'] == "Rome")|(_opts['batch'] == "local" ):
     _executable = "sub_%s_%s"%(_opts['mode'],_opts['ext'])
 
     # Write details depending on mode
@@ -102,8 +102,9 @@ def submitFiles(_opts):
     print "  --> Finished submitting files"
 
   # SGE
-  elif _opts['batch'] in ['IC','SGE']:
+  elif _opts['batch'] in ['IC','SGE','Rome']:
     _executable = "sub_%s_%s"%(_opts['mode'],_opts['ext'])
+    _subcmd = 'bsub' if _opts['batch']=='Rome' else 'qsub'
 
     # Extract job opts
     jobOptsStr = _opts['jobOpts']
@@ -113,7 +114,7 @@ def submitFiles(_opts):
       for cidx in range(_opts['nCats']):
         c = _opts['cats'].split(",")[cidx]
         _subfile = "%s/%s_%s"%(_jobdir,_executable,c)
-        cmdLine = "qsub -q hep.q %s -o %s.log -e %s.err %s.sh"%(jobOptsStr,_subfile,_subfile,_subfile)
+        cmdLine = "%s -q %s %s -o %s.log -e %s.err %s.sh"%(_subcmd,_opts['queue'],jobOptsStr,_subfile,_subfile,_subfile)
         run(cmdLine)
     print "  --> Finished submitting files"
 

Combine/Checks/Bias_nominal/BiasFits

@@ -0,0 +1 @@
+/eos/home-f/fderiggi/AC/Bias_study/fit