fix: OutOfMemoryError for FTOF and EPICS (#152)

bin/environ.sh

@@ -42,8 +42,12 @@ timeline_groovy_opts=(
   -Djava.awt.headless=true
 )
 
+# run java with more resources, to mitigate large memory residence for long run periods
+timeline_java_opts_highmem=$(echo ${timeline_java_opts[*]} | sed 's;Xmx.*m;Xmx2048m;')
+
 # exports
 export CLASSPATH="$(echo "${java_classpath[*]}" | sed 's; ;:;g')${CLASSPATH:+:${CLASSPATH}}"
 export JYPATH="$(echo "${groovy_classpath[*]}" | sed 's; ;:;g')${JYPATH:+:${JYPATH}}"
 export TIMELINE_JAVA_OPTS="${timeline_java_opts[*]}"
 export TIMELINE_GROOVY_OPTS="${timeline_groovy_opts[*]}"
+export TIMELINE_JAVA_OPTS_HIGHMEM=$timeline_java_opts_highmem

detectors/src/main/java/org/jlab/clas/timeline/fitter/FTOFFitter.groovy

@@ -31,33 +31,33 @@ class FTOFFitter {
     return f1
   }
 
-	static F1D timefit_p2(H1F h1) {
-	    def f1 = new F1D("fit:"+h1.getName(), "[amp]*gaus(x,[mean],[sigma])", -0.5, 0.5);
-        double hAmp  = h1.getBinContent(h1.getMaximumBin());
-        double hMean = h1.getAxis().getBinCenter(h1.getMaximumBin());
-        double hRMS  = 0.3;//h1.getRMS(); //ns
-        double rangeMin = (hMean - 2.5*hRMS);
-        double rangeMax = (hMean + 2.5*hRMS);
-        f1.setRange(rangeMin, rangeMax);
-        f1.setParameter(0, hAmp);
-        f1.setParameter(1, hMean);
-        f1.setParameter(2, hRMS);
-
-		def makefit = {func->
-		  hMean = func.getParameter(1)
-		  hRMS = func.getParameter(2).abs()
-		  func.setRange(hMean-1.5*hRMS,hMean+1.5*hRMS)
-		  MoreFitter.fit(func,h1,"Q")
-		  return [func.getChiSquare(), (0..<func.getNPars()).collect{func.getParameter(it)}]
-		}
-
-		def fits1 = (0..20).collect{makefit(f1)}
-		def bestfit = fits1.sort()[0]
-		f1.setParameters(*bestfit[1])
-		//makefit(f1)
-		return f1
-	}
 
+  static F1D timefit_p2(H1F h1) {
+    def f1 = new F1D("fit:"+h1.getName(), "[amp]*gaus(x,[mean],[sigma])", -0.5, 0.5);
+    double hAmp  = h1.getBinContent(h1.getMaximumBin());
+    double hMean = h1.getAxis().getBinCenter(h1.getMaximumBin());
+    double hRMS  = 0.3;//h1.getRMS(); //ns
+    double rangeMin = (hMean - 2.5*hRMS);
+    double rangeMax = (hMean + 2.5*hRMS);
+    f1.setRange(rangeMin, rangeMax);
+    f1.setParameter(0, hAmp);
+    f1.setParameter(1, hMean);
+    f1.setParameter(2, hRMS);
+
+    def makefit = {func->
+      hMean = func.getParameter(1)
+      hRMS = func.getParameter(2).abs()
+      func.setRange(hMean-1.5*hRMS,hMean+1.5*hRMS)
+      MoreFitter.fit(func,h1,"Q")
+      return [func.getChiSquare(), (0..<func.getNPars()).collect{func.getParameter(it)}]
+    }
+
+    def fits1 = (0..20).collect{makefit(f1)}
+    def bestfit = fits1.sort()[0]
+    f1.setParameters(*bestfit[1])
+    //makefit(f1)
+    return f1
+  }
 
 
   static F1D tdcadcdifffit_p1a(H1F h1) {
@@ -72,7 +72,6 @@ class FTOFFitter {
     f1.setParameter(1, hMean);
     f1.setParameter(2, hRMS);
 
-
     def makefit = {func->
       hMean = func.getParameter(1)
       hRMS = func.getParameter(2).abs()
@@ -88,6 +87,7 @@ class FTOFFitter {
     return f1
   }
 
+
   static F1D tdcadcdifffit_p1b(H1F h1) {
     def f1 = new F1D("fit:"+h1.getName(), "[amp]*gaus(x,[mean],[sigma])", -1.0, 1.0);
     double hAmp  = h1.getBinContent(h1.getMaximumBin());
@@ -120,27 +120,26 @@ class FTOFFitter {
     def f1 = new F1D("fit:"+h1.getName(), "[amp]*gaus(x,[mean],[sigma])", -1.0, 1.0);
     double hAmp  = h1.getBinContent(h1.getMaximumBin());
     double hMean = h1.getAxis().getBinCenter(h1.getMaximumBin());
-    double hRMS  = h1.getRMS(); //ns
-    double rangeMin = (hMean - 2.5*hRMS);
-    double rangeMax = (hMean + hRMS);
+    double hRMS  = 1.9//h1.getRMS(); //ns
+    double rangeMin = (hMean - 1.5*hRMS);
+    double rangeMax = (hMean + 1.5*hRMS);
     f1.setRange(rangeMin, rangeMax);
     f1.setParameter(0, hAmp);
     f1.setParameter(1, hMean);
     f1.setParameter(2, hRMS);
 
-
     def makefit = {func->
       hMean = func.getParameter(1)
       hRMS = func.getParameter(2).abs()
-      func.setRange(hMean - 1.5*hRMS, hMean + 1.5*hRMS)
+      func.setRange(hMean-1.5*hRMS,hMean+1.5*hRMS)
       MoreFitter.fit(func,h1,"Q")
       return [func.getChiSquare(), (0..<func.getNPars()).collect{func.getParameter(it)}]
     }
 
     def fits1 = (0..20).collect{makefit(f1)}
     def bestfit = fits1.sort()[0]
     f1.setParameters(*bestfit[1])
-    f1.setRange(rangeMin, rangeMax)
+    //makefit(f1)
     return f1
   }
 

detectors/src/main/java/org/jlab/clas/timeline/fitter/MoreFitter.groovy

@@ -21,42 +21,6 @@ class MoreFitter {
     }
   }
 
-  static F1D gausFit(H1F h1, String opts) {
-    def f1 = new F1D('fit:'+h1.getName(), '[amp]*gaus(x,[mean],[sigma])', 0,1)
-
-    def scale = Math.max(h1.getBinContent(h1.getMaximumBin())/500, 1)
-    def data = (1..<h1.getDataSize(0)).collectMany{[h1.getDataX(it)]*(h1.getBinContent(it)/scale as int)}
-    int nlen = data.size()
-    if(nlen==0) {
-      f1.setParameters(0,h1.getDataX(0),0)
-      return f1
-    }
-
-    def (q1,q2,q3) = [data[nlen/4 as int], data[nlen/2 as int], data[nlen*3/4 as int]]
-    def (x1,x2) = [1, h1.getDataSize(0)-1].collect{h1.getDataX(it)}
-    def norms = [h1.getBinContent(h1.getMaximumBin())]
-    def mus = [h1.getAxis().getBinCenter(h1.getMaximumBin()), h1.getMean(), q2, h1.getDataX(h1.getDataSize(0)/2 as int)]
-    def sigs = [h1.getRMS()/2, (q3-q1)/1.35]
-
-    def makefit = {amp,mu,sig->
-      f1.setParameters(amp,mu,sig)
-      def rng = [mu-2.5*sig, mu+2.5*sig]
-      f1.setRange(*rng)
-      DataFitter.fit(f1,h1,"Q")
-      (amp,mu,sig) = (0..<f1.getNPars()).collect{f1.getParameter(it)}
-      if(mu<x1 || mu>x2 || sig>(x2-x1)) return null
-      return [f1.getChiSquare(), [amp,mu,sig], rng]
-    }
-
-    def fits = [norms,mus,sigs].combinations().findResults{makefit(it)}
-    fits += fits.findResults{makefit(it[1])}
-
-    def best = fits.min{it[0]} ?: [0, [0,x1,x2-x1], [x1,x2]]
-    f1.setParameters(*best[1])
-    f1.setRange(*best[2])
-    return f1
-  }
-
   static F1D fitgaus(H1F h1) {
     def f1 = new F1D('fit:'+h1.getName(), '[amp]*gaus(x,[mean],[sigma])', 0,1)
     f1.setRange(h1.getDataX(0), h1.getDataX(h1.getDataSize(0)-1))

detectors/src/main/java/org/jlab/clas/timeline/run.groovy

@@ -100,9 +100,6 @@ def engines = [
     new ftof.ftof_tdcadc_p1a(),
     new ftof.ftof_tdcadc_p1b(),
     new ftof.ftof_tdcadc_p2(),
-    // new ftof.ftof_tdcadc_p1a_zoomed(),
-    // new ftof.ftof_tdcadc_p1b_zoomed(),
-    // new ftof.ftof_tdcadc_p2_zoomed(),
     new ftof.ftof_ctof_vtdiff(),
     new dc.dc_residuals_sec(),
     new dc.dc_residuals_sec_sl(),

detectors/src/main/java/org/jlab/clas/timeline/run_rgb.groovy

@@ -103,9 +103,6 @@ def engines = [
     new ftof.ftof_tdcadc_p1a(),
     new ftof.ftof_tdcadc_p1b(),
     new ftof.ftof_tdcadc_p2(),
-    // new ftof.ftof_tdcadc_p1a_zoomed(),
-    // new ftof.ftof_tdcadc_p1b_zoomed(),
-    // new ftof.ftof_tdcadc_p2_zoomed(),
     new ftof.ftof_ctof_vtdiff(),
     new dc.dc_residuals_sec(),
     new dc.dc_residuals_sec_sl(),

detectors/src/main/java/org/jlab/clas/timeline/timeline/epics/MYQuery.groovy

@@ -14,6 +14,7 @@ class MYQuery {
     'd': 'on',          // data update events only
     'p': 'on',          // include prior point
     'm': 'history',     // MYA deployment
+    'l': '',            // limit by binning (downsample)
   ]
 
   // timestamps

detectors/src/main/java/org/jlab/clas/timeline/timeline/epics/epics_xy.groovy

@@ -3,6 +3,7 @@ package org.jlab.clas.timeline.timeline.epics
 import java.text.SimpleDateFormat
 import org.jlab.groot.data.TDirectory
 import org.jlab.groot.data.H1F
+import org.jlab.groot.math.F1D
 import org.jlab.groot.data.GraphErrors
 import org.jlab.clas.timeline.fitter.MoreFitter
 
@@ -14,9 +15,49 @@ class epics_xy {
     runlist.push(run)
   }
 
+  static F1D gausFit(H1F h1) { // FIXME: not stable...
+
+    def f1 = new F1D("fit:"+h1.getName(), "[amp]*gaus(x,[mean],[sigma])", -10, 10);
+    double hAmp  = h1.getBinContent(h1.getMaximumBin());
+    double hMean = h1.getAxis().getBinCenter(h1.getMaximumBin());
+    double hRMS  = Math.min(h1.getRMS(),1.0);
+    f1.setRange(hMean-2.0*hRMS, hMean+2.0*hRMS);
+    f1.setParameter(0, hAmp);
+    f1.setParameter(1, hMean);
+    f1.setParameter(2, hRMS);
+
+    def makefit = {func->
+      hMean = func.getParameter(1)
+      hRMS = func.getParameter(2).abs()
+      func.setRange(hMean-2.0*hRMS,hMean+2.0*hRMS)
+      MoreFitter.fit(func,h1,"Q")
+      return [func.getChiSquare(), (0..<func.getNPars()).collect{func.getParameter(it)}]
+    }
+
+    def fits1 = (0..20).collect{makefit(f1)}
+    def bestfit = fits1.sort()[0]
+    f1.setParameters(*bestfit[1])
+    return f1
+
+  }
+
+  static F1D justUseStats(H1F h1) { // no fit, just use mean and RMS from the histogram
+    def f1 = new F1D("fit:"+h1.getName(), "[amp]*gaus(x,[mean],[sigma])", -10, 10);
+    double hAmp  = h1.getBinContent(h1.getMaximumBin());
+    double hMean = h1.getMean()
+    double hRMS  = h1.getRMS()
+    f1.setRange(hMean-2.0*hRMS, hMean+2.0*hRMS);
+    f1.setParameter(0, hAmp);
+    f1.setParameter(1, hMean);
+    f1.setParameter(2, hRMS);
+    return f1
+  }
+
+
   def close() {
 
     def MYQ = new MYQuery()
+    MYQ.querySettings['l'] = "${1000*runlist.size()}" // downsample the payload, since it's too big for a full run period
     def ts = MYQ.getRunTimeStamps(runlist)
 
     def epics = [:].withDefault{[:]}
@@ -69,12 +110,13 @@ class epics_xy {
         hy.fill(it[2], it[0]*it[3]/1000)
       }
 
-      def fx = MoreFitter.gausFit(hx, "Q")
-      def fy = MoreFitter.gausFit(hy, "Q")
+      def fx = justUseStats(hx)
+      def fy = justUseStats(hy)
       grx.addPoint(run, fx.getParameter(1), 0,0)
       gry.addPoint(run, fy.getParameter(1), 0,0)
 
-      [hx,hy,fx,fy].each{out.addDataSet(it)}
+      [hx,hy].each{out.addDataSet(it)}
+      // [fx,fy].each{out.addDataSet(it)}
       println("$run done")
     }
 

detectors/src/main/java/org/jlab/clas/timeline/timeline/ftof/ftof_tdcadc_p1a.groovy

@@ -3,6 +3,7 @@ import java.util.concurrent.ConcurrentHashMap
 import org.jlab.groot.data.TDirectory
 import org.jlab.groot.data.GraphErrors
 import org.jlab.clas.timeline.fitter.FTOFFitter
+import org.jlab.clas.timeline.util.HistoUtil
 
 class ftof_tdcadc_p1a {
 
@@ -14,7 +15,7 @@ def processDirectory(dir, run) {
   def sigmalist = []
   def chi2list = []
   def histlist =   (0..<6).collect{
-    def h1 = dir.getObject('/tof/p1a_tdcadc_dt_S'+(it+1))
+    def h1 = HistoUtil.zoomHisto(dir.getObject('/tof/p1a_tdcadc_dt_S'+(it+1)))
     def f1 = FTOFFitter.tdcadcdifffit_p1a(h1)
 
     funclist.add(f1)

detectors/src/main/java/org/jlab/clas/timeline/timeline/ftof/ftof_tdcadc_p1b.groovy

@@ -3,6 +3,7 @@ import java.util.concurrent.ConcurrentHashMap
 import org.jlab.groot.data.TDirectory
 import org.jlab.groot.data.GraphErrors
 import org.jlab.clas.timeline.fitter.FTOFFitter
+import org.jlab.clas.timeline.util.HistoUtil
 
 class ftof_tdcadc_p1b {
 
@@ -14,7 +15,7 @@ def processDirectory(dir, run) {
   def sigmalist = []
   def chi2list = []
   def histlist =   (0..<6).collect{
-    def h1 = dir.getObject('/tof/p1b_tdcadc_dt_S'+(it+1))
+    def h1 = HistoUtil.zoomHisto(dir.getObject('/tof/p1b_tdcadc_dt_S'+(it+1)))
     def f1 = FTOFFitter.tdcadcdifffit_p1b(h1)
 
     funclist.add(f1)