diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java index c21fb2622..f3617a141 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/ADCTDCMerger.java @@ -1,7 +1,6 @@ package org.jlab.analysis.eventmerger; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import org.jlab.detector.banks.RawBank; @@ -28,8 +27,8 @@ public class ADCTDCMerger { private OrderType[] selectedOrders = {OrderType.NOMINAL, OrderType.BGADDED_NOMINAL, OrderType.BGREMOVED}; private EventMergerConstants constants; - private DataEvent event; - private List bgEvents; + private DataEvent event; + private List[] bgEvents; private int run; /** @@ -39,13 +38,12 @@ public class ADCTDCMerger { * @param signal physics events * @param bgs background events */ - public ADCTDCMerger(EventMergerConstants constants, DataEvent signal, DataEvent... bgs) { + public ADCTDCMerger(EventMergerConstants constants, DataEvent signal, List... bgs) { this.constants = constants; this.event = signal; - this.bgEvents = new ArrayList<>(); - this.bgEvents.addAll(Arrays.asList(bgs)); - if(!bgEvents.isEmpty() && bgEvents.get(0).hasBank("RUN::config")) - run = bgEvents.get(0).getBank("RUN::config").getInt("run", 0); + this.bgEvents = bgs; + if(bgEvents.length!=0 && !bgEvents[0].isEmpty() && bgEvents[0].get(0).hasBank("RUN::config")) + run = bgEvents[0].get(0).getBank("RUN::config").getInt("run", 0); } /** @@ -122,56 +120,69 @@ public List readTDCs(DetectorType detector, DataBank bank) { } /** - * Merge ADC banks for data (signal) and background events for selected detector + * Merge ADC banks for data (signal) and background events for the selected detector and layers * In case of multiple hit on same detector element, only first hit in time is kept * unless the double-hit suppression flag, suppressDoubleHits, is set to false * * @param detector + * @param layers * @return */ - public DataBank mergeADCs(DetectorType detector){ + public DataBank mergeADCs(DetectorType detector, List layers){ - DataEvent bg = bgEvents.get(0); + List bgs = bgEvents[0]; String ADCString = detector.getName()+"::adc"; - if(!bg.hasBank(ADCString)) { - return event.getBank(ADCString); + for(DataEvent bg : bgs) { + if(!bg.hasBank(ADCString)) { + return event.getBank(ADCString); + } } - else { - List bgADCs = readADCs(detector,bg.getBank(ADCString)); - List ADCs = readADCs(detector,event.getBank(ADCString)); - - DataBank bank = this.writeToBank(event, ADCString, this.merge(ADCs, bgADCs)); - return bank; + List bgADCs = new ArrayList<>(); + for(DataEvent bg : bgs) { + for(DGTZ dgtz : readADCs(detector,bg.getBank(ADCString))) { + if(layers==null || layers.contains(dgtz.getLayer())) + bgADCs.add((ADC) dgtz); + } } + List ADCs = readADCs(detector,event.getBank(ADCString)); + + DataBank bank = this.writeToBank(event, ADCString, this.merge(ADCs, bgADCs)); + + return bank; + } /** - * Merge TDC banks for data (signal) and background events for selected detector + * Merge TDC banks for data (signal) and background events for the selected detector and layers * Use two background events shifted in time to extend the time range of the backgrounds * Multiple hits on the same components are kept if time distance exceed the holdoff time * * @param detector + * @param layers * @return */ - public DataBank mergeTDCs(DetectorType detector){ + public DataBank mergeTDCs(DetectorType detector, List layers){ + List bgs = bgEvents[0]; + String TDCString = detector+"::tdc"; // if the primary background event has no detector bank then keep the event bank - if(!bgEvents.get(0).hasBank(TDCString)) { - return event.getBank(TDCString); - } - // if the primary background events has the detector bank, then proceed with merging - else { - // get background hits using multiple events dependending on detector - int bgSize = constants.getInt(detector, EventMergerEnum.MERGE_SIZE); - if(!event.hasBank(TDCString)) bgSize = 1; - // collect bg hits - List bgTDCs = new ArrayList<>(); - for(int i=0; i bgTDCs = new ArrayList<>(); + for(int i=0; i TDCs = readTDCs(detector, event.getBank(TDCString)); - - // merge physics and bg hit - List mergedTDCs = this.merge(TDCs, bgTDCs); - - // create output bank - return this.writeToBank(event, TDCString, mergedTDCs); } + + // get physics event hits hits + List TDCs = readTDCs(detector, event.getBank(TDCString)); + + // merge physics and bg hit + List mergedTDCs = this.merge(TDCs, bgTDCs); + + // create output bank + return this.writeToBank(event, TDCString, mergedTDCs); + } /** @@ -487,11 +501,22 @@ public DGTZ(DetectorType detector) { public DGTZ(DetectorType detector, byte sector, byte layer, short component, byte order) { super(detector); this.setSectorLayerComponent(sector, layer, component); - this.setOrder(order); - this.removed = false; - this.background = false; + this.setOrderAndType(order); + } + + public void setOrderAndType(int order) { + OrderType type = OrderType.getType(order); + this.setOrder(order - type.getTypeId()); + if(type==OrderType.BGREMOVED) + this.removed = true; + if(type==OrderType.BGADDED_NOMINAL) + this.background = true; + if(type==OrderType.BGREMOVED_BG) { + this.removed = true; + this.background = true; + } } - + public boolean isGood() { return true; } @@ -515,8 +540,6 @@ public void markAsBackground() { public boolean status() { if(!this.isInTime()) return false; - else if(this.isBackground() && this.isRemoved()) - return false; else { for(OrderType o : selectedOrders) { if(this.getOrderType()==o) @@ -527,12 +550,14 @@ else if(this.isBackground() && this.isRemoved()) } public RawBank.OrderType getOrderType() { - if(this.isBackground()) - return RawBank.OrderType.BGADDED_NOMINAL; + if(this.isBackground() && this.isRemoved()) + return OrderType.BGREMOVED_BG; + else if(this.isBackground()) + return OrderType.BGADDED_NOMINAL; else if(this.isRemoved()) - return RawBank.OrderType.BGREMOVED; + return OrderType.BGREMOVED; else - return RawBank.OrderType.NOMINAL; + return OrderType.NOMINAL; } @@ -553,7 +578,7 @@ public void readFromBank(DataBank bank, int row) { short component = bank.getShort("component", row); byte order = bank.getByte("order", row); this.setSectorLayerComponent(sector, layer, component); - this.setOrder(order); + this.setOrderAndType(order); } public boolean pilesUp(DGTZ o){ diff --git a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java index 992516ae7..c37452c3a 100644 --- a/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java +++ b/common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java @@ -1,7 +1,10 @@ package org.jlab.analysis.eventmerger; +import java.io.File; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import org.jlab.detector.banks.RawBank; @@ -35,13 +38,18 @@ public class EventMerger { private boolean preserveHitOrder = true; private EventMergerConstants constants = new EventMergerConstants(); - private List detectors; + private Map> detectors; private OrderType[] orders; - + private List bgFileNames; + private boolean reuseBgEvents = false; + private int bgScale = 1; + private int bgFileIndex = 0; + private HipoDataSource bgReader; + public EventMerger() { - detectors = Arrays.asList(DetectorType.DC, DetectorType.FTOF); - orders = this.getOrders(OrderType.NOMINAL.name(),OrderType.BGADDED_NOMINAL.name(),OrderType.BGREMOVED.name()); + detectors = this.getDetectors(DetectorType.DC.getName(), DetectorType.FTOF.getName()); + orders = this.getOrders(OrderType.NOMINAL.name(),OrderType.BGADDED_NOMINAL.name(),OrderType.BGREMOVED.name(),OrderType.BGREMOVED_BG.name()); printConfiguration(); } @@ -53,16 +61,41 @@ public EventMerger(String[] dets, String[] types, boolean dhits, boolean ohits) printConfiguration(); } - private List getDetectors(String[] dets) { - List all = new ArrayList<>(); + private Map> getDetectors(String... dets) { + Map> all = new HashMap<>(); if(dets.length==1 && dets[0].equals("ALL")) { - all.addAll(EventMergerConstants.ADCs); - for(DetectorType t : EventMergerConstants.TDCs) { - if(!all.contains(t)) all.add(t); + for(DetectorType t : EventMergerConstants.ADCs) { + all.put(t, null); + } + for(DetectorType d : EventMergerConstants.TDCs) { + if(!all.containsKey(d)) all.put(d, null); } } else { - for(String d : dets) all.add(DetectorType.getType(d)); + for(String d : dets) { + String[] dn = d.split("(?<=\\D)(?=\\d)"); + DetectorType type = dn.length>0 ? DetectorType.getType(dn[0]) : DetectorType.UNDEFINED; + if(type == DetectorType.UNDEFINED) { + throw new IllegalArgumentException("Unknown detector type " + type); + } + else if(dn.length==2 && type==DetectorType.DC && dn[1].matches("[1-3]+")) { + int region = Integer.parseInt(dn[1]); + if(!all.containsKey(DetectorType.DC)) + all.put(DetectorType.DC, new ArrayList<>()); + for(int il=0; il<12; il++) { + int layer = (region-1)*12+il+1; + if(!all.get(DetectorType.DC).contains(layer)) + all.get(DetectorType.DC).add(layer); + } + } + else { + all.put(type, null); + } + } + for(DetectorType type : all.keySet()) { + if(all.get(type)!=null) + Collections.sort(all.get(type)); + } } return all; } @@ -80,6 +113,51 @@ private OrderType[] getOrders(String... type) { return null; } + public boolean setBgFiles(List filenames, int scale, boolean reuse) { + bgFileNames = new ArrayList<>(); + for (String filename : filenames) { + File f = new File(filename); + if (!f.exists() || !f.isFile() || !f.canRead()) { + Logger.getLogger(EventMerger.class.getName()).log(Level.SEVERE,"Background filename {0} invalid.",filename); + return false; + } + Logger.getLogger(EventMerger.class.getName()).log(Level.INFO,"Background files: reading {0}",filename); + bgFileNames.add(filename); + } + if(scale>0) bgScale = scale; + reuseBgEvents = reuse; + return true; + } + + private boolean openNextFile() { + if(bgFileIndex == bgFileNames.size()) { + if (reuseBgEvents) { + Logger.getLogger(EventMerger.class.getName()).info("Reopening previously used background file"); + bgFileIndex = 0; + } + else { + Logger.getLogger(EventMerger.class.getName()).info("Ran out of background events"); + return false; + } + } + bgReader = new HipoDataSource(); + bgReader.open(bgFileNames.get(bgFileIndex)); + bgFileIndex++; + return true; + } + + synchronized public List getBackgroundEvents(int n) { + List events = new ArrayList<>(); + for(int i=0; i bgs) { - if(!event.hasBank("RUN::config") || !bg1.hasBank("RUN::config") || !bg2.hasBank("RUN::config")) { + if(!event.hasBank("RUN::config")) return; - } + if(bgs.isEmpty() || bgs.size()%2!=0) + return; + for(DataEvent bg : bgs) + if(!bg.hasBank("RUN::config")) + return; if(event.hasBank("DC::doca")) event.removeBank("DC::doca"); + int nbg = bgs.size()/2; + List bg1 = bgs.subList(0, nbg); + List bg2 = bgs.subList(nbg, 2*nbg); ADCTDCMerger merger = new ADCTDCMerger(constants, event, bg1, bg2); merger.setSuppressDoubleHits(suppressDoubleHits); merger.setPreserveHitOrder(preserveHitOrder); merger.setSelectedOrders(orders); - for(DetectorType det : detectors) { + for(DetectorType det : detectors.keySet()) { List banks = new ArrayList<>(); List names = new ArrayList<>(); + List layers = detectors.get(det); + if(EventMergerConstants.ADCs.contains(det)) { names.add(det.getName()+"::adc"); - banks.add(merger.mergeADCs(det)); + banks.add(merger.mergeADCs(det, layers)); } if(EventMergerConstants.TDCs.contains(det)) { names.add(det.getName()+"::tdc"); - banks.add(merger.mergeTDCs(det)); + banks.add(merger.mergeTDCs(det, layers)); } if(banks.isEmpty()) System.out.println("Unknown detector:" + det); @@ -141,62 +235,88 @@ public void mergeEvents(DataEvent event, DataEvent bg1, DataEvent bg2) { } } + /** + * Append merged banks to hipo event + * + * @param event + * @return + */ + public boolean mergeEvents(DataEvent event) { + + return this.mergeEvents(event, bgScale); + } + + + private boolean mergeEvents(DataEvent event, int scale) { + + List eventsBg = this.getBackgroundEvents(2*scale); + + if(eventsBg==null) return false; + + this.mergeEvents(event,eventsBg); + return true; + } + public static void main(String[] args) { DefaultLogger.debug(); OptionParser parser = new OptionParser("bg-merger"); parser.addRequired("-o" ,"merged file"); - parser.addRequired("-i" ,"input data file"); - parser.addRequired("-b" ,"background file"); - parser.setRequiresInputList(false); + parser.addRequired("-i" ,"signal event file"); + parser.setRequiresInputList(true); parser.addOption("-n" ,"-1", "maximum number of events to process"); - parser.addOption("-d" ,"DC,FTOF", "list of detectors, for example \"DC,FTOF,HTCC\" or \"ALL\" for all available detectors"); + parser.addOption("-d" ,"ALL", "list of detectors, for example \"DC,FTOF,HTCC\" or \"ALL\" for all available detectors. Use DC1, DC2 or DC3 to select the DC region"); + parser.addOption("-r" ,"1", "reuse background events: 0-false, 1-true"); parser.addOption("-s" ,"1", "suppress double TDC hits on the same component, 0-no suppression, 1-suppression"); parser.addOption("-l" ,"1", "preserve initial hit order (for compatibility with truth matching, 0-false, 1-true"); - parser.addOption("-t" ,"NOMINAL,BGADDED_NOMINAL,BGREMOVED", "list of hit OrderTypes to be saved"); + parser.addOption("-t" ,"NOMINAL,BGADDED_NOMINAL,BGREMOVED,BGREMOVED_BG", "list of hit OrderTypes to be saved"); + parser.addOption("-x" ,"1", "background scale factor"); parser.parse(args); if(parser.hasOption("-i") && parser.hasOption("-o")){ String dataFile = parser.getOption("-i").stringValue(); String outputFile = parser.getOption("-o").stringValue(); - String bgFile = parser.getOption("-b").stringValue(); + List bgFiles = parser.getInputList(); int maxEvents = parser.getOption("-n").intValue(); String detectors = parser.getOption("-d").stringValue(); String ordertypes = parser.getOption("-t").stringValue(); boolean doubleHits = (parser.getOption("-s").intValue()==1); + int bgScale = parser.getOption("-x").intValue(); + boolean reuseBG = (parser.getOption("-r").intValue()==1); boolean hitOrder = (parser.getOption("-l").intValue()==1); + EventMerger merger = new EventMerger(detectors.split(","),ordertypes.split(","),doubleHits,hitOrder); - + if(!merger.setBgFiles(bgFiles, bgScale, reuseBG)) + System.exit(1); + int counter = 0; - // Readers for event and background + // Reader for signal events HipoDataSource readerData = new HipoDataSource(); readerData.open(dataFile); - HipoDataSource readerBg = new HipoDataSource(); - readerBg.open(bgFile); - + //Writer HipoDataSync writer = new HipoDataSync(); writer.setCompressionType(2); writer.open(outputFile); ProgressPrintout progress = new ProgressPrintout(); - while (readerData.hasEvent()&& readerBg.hasEvent()) { + while (readerData.hasEvent()) { counter++; //System.out.println("************************************************************* "); DataEvent eventData = readerData.getNextEvent(); - DataEvent eventBg1 = readerBg.getNextEvent(); - if(!readerBg.hasEvent()) break; - DataEvent eventBg2 = readerBg.getNextEvent(); - merger.mergeEvents(eventData, eventBg1, eventBg2); - writer.writeEvent(eventData); + if(merger.mergeEvents(eventData)) + writer.writeEvent(eventData); + else + maxEvents = counter; + progress.updateStatus(); if(maxEvents>0){ if(counter>=maxEvents) break; diff --git a/common-tools/clas-detector/src/main/java/org/jlab/detector/banks/RawBank.java b/common-tools/clas-detector/src/main/java/org/jlab/detector/banks/RawBank.java index da1c74b09..fffc360b5 100644 --- a/common-tools/clas-detector/src/main/java/org/jlab/detector/banks/RawBank.java +++ b/common-tools/clas-detector/src/main/java/org/jlab/detector/banks/RawBank.java @@ -67,7 +67,7 @@ public static enum OrderType { NOMINAL ( 0), // normal hits retained by denoising level-0 BGADDED_NOMINAL ( 10), // hits added by background merging and retained by level-0 BGREMOVED ( 20), // hits removed during background merging - RESERVED ( 30), // reserved for later use + BGREMOVED_BG ( 30), // background hits removed by background merging NOISE1 ( 40), // normal hits retained by level-1 denoising NOISE2 ( 50), // normal hits retained by level-2 denoising NOISE3 ( 60), // normal hits retained by level-3 denoising @@ -80,8 +80,14 @@ public static enum OrderType { private final int rawOrderId; private OrderType(int id){ rawOrderId = id; } public int getTypeId() { return rawOrderId; } + public static OrderType getType(int order) { + for(OrderType t : OrderType.values()) + if(t.rawOrderId == order-order%10) + return t; + return null; + } } - + public static final String FILTER_VAR_NAME = "order"; public RawBank(Schema sch){ diff --git a/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java b/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java index 9ca9e9406..1cec83e1e 100644 --- a/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java +++ b/reconstruction/bg/src/main/java/org/jlab/service/bg/BackgroundEngine.java @@ -1,6 +1,7 @@ package org.jlab.service.bg; import java.io.File; +import java.util.Arrays; import java.util.LinkedList; import java.util.logging.Level; import java.util.logging.Logger; @@ -21,14 +22,12 @@ public class BackgroundEngine extends ReconstructionEngine { public static final String CONF_SUPPRESS_DOUBLES = "suppressDoubles"; public static final String CONF_PRESERVE_ORDER = "preserveOrder"; public static final String CONF_REUSE_EVENTS = "reuseEvents"; + public static final String CONF_BG_SCALE = "bgScale"; static final Logger logger = Logger.getLogger(BackgroundEngine.class.getName()); - int filesUsed = 0; - boolean reuseEvents = true; EventMerger bgmerger = null; - HipoDataSource bgreader = null; - LinkedList bgfilenames = new LinkedList<>(); +// LinkedList bgfilenames = new LinkedList<>(); public BackgroundEngine() { super("BG", "baltzell", "1.0"); @@ -42,51 +41,19 @@ public boolean init() { } public boolean init(String... filenames) { - bgfilenames.clear(); String detectors = getEngineConfigString(CONF_DETECTORS,"DC,FTOF"); String orders = getEngineConfigString(CONF_ORDERS,"NOMINAL"); boolean suppressDoubles = Boolean.valueOf(getEngineConfigString(CONF_SUPPRESS_DOUBLES,"true")); boolean preserveOrder = Boolean.valueOf(getEngineConfigString(CONF_PRESERVE_ORDER,"true")); boolean reuseEvents = Boolean.valueOf(getEngineConfigString(CONF_REUSE_EVENTS,"false")); - for (String filename : filenames) { - File f = new File(filename); - if (!f.exists() || !f.isFile() || !f.canRead()) { - logger.log(Level.SEVERE,"BackgroundEngine:: filename {0} invalid.",filename); - return false; - } - logger.log(Level.INFO,"BackgroundEngine:: reading {0}",filename); - bgfilenames.add(filename); - } + int bgScale = Integer.valueOf(getEngineConfigString(CONF_BG_SCALE,"1")); bgmerger = new EventMerger(detectors.split(","), orders.split(","), suppressDoubles, preserveOrder); - openNextFile(); - return true; - } - - private void openNextFile() { - if (filesUsed>0 && filesUsed%bgfilenames.size()==0) { - if (reuseEvents) logger.info("BackgroundEngine:: Reopening previously used file."); - else throw new RuntimeException("BackgroundEngine:: Ran out of events."); - } - String filename = bgfilenames.remove(); - bgfilenames.add(filename); - bgreader = new HipoDataSource(); - bgreader.open(filename); - filesUsed++; - } - - synchronized public DataEvent getBackgroundEvent() { - if (!bgreader.hasEvent()) openNextFile(); - return bgreader.getNextEvent(); + return bgmerger.setBgFiles(Arrays.asList(filenames), bgScale, reuseEvents); } @Override public boolean processDataEvent(DataEvent event) { - if (!bgfilenames.isEmpty()) { - DataEvent a = getBackgroundEvent(); - DataEvent b = getBackgroundEvent(); - bgmerger.mergeEvents(event, a, b); - } - return true; + return bgmerger.mergeEvents(event); } }