Merge pull request #207 from fact-project/SinglePulseGainCalibrationS…

…ervice A Single pulse gain calibration service to solve issue #197
fact-project · Jan 25, 2017 · 039d26a · 039d26a
2 parents 6359d71 + ba86cd8
commit 039d26a
Show file tree

Hide file tree

Showing 29 changed files with 160 additions and 121 deletions.
diff --git a/examples/apply_model.xml b/examples/apply_model.xml
@@ -12,7 +12,7 @@
 
     <property name="auxFolder" value="file:src/main/resources/aux/" />
     <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
-
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
     <!--In case you want to analyze .zfits files.-->

diff --git a/examples/fellwalker_example.xml b/examples/fellwalker_example.xml
@@ -10,6 +10,7 @@
     <property name="outfile" value="file:/home/fabian/testoutfile_mc.txt" />
 
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
 
     <stream id="fact" class="fact.io.FITSStream" url="${infile}"/>
 

diff --git a/examples/forErna/std_analysis_erna.xml b/examples/forErna/std_analysis_erna.xml
@@ -11,6 +11,7 @@
     <property name="pixelDelayFile" value="classpath:/default/delays_lightpulser_20150217.csv" />
 
     <service id="auxService" class="fact.auxservice.SqliteService" url="${db}" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
     <!-- Has to be a  FactFileListMultiStream in order to work on tasks from gridmap and the executor script.-->

diff --git a/examples/forErna/std_analysis_mc_erna.xml b/examples/forErna/std_analysis_mc_erna.xml
@@ -8,6 +8,7 @@
     <property name="integralGainFile" value="classpath:/default/defaultIntegralGains.csv" />
 
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
 
     <!-- Has to be a  FactFileListMultiStream in order to work on tasks from gridmap and the executor script.-->
     <stream id="fact" class="fact.io.FactFileListMultiStream"   url="${input}">

diff --git a/examples/mc_viewer.xml b/examples/mc_viewer.xml
@@ -8,6 +8,7 @@
 
 	<!-- The calibration service delivers aux data to the processors -->
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
 
   <!-- Name of the stream and url to the input file -->
   <stream id="fact" class="fact.io.FITSStream" url="${infile}" />

diff --git a/examples/measure_performance.xml b/examples/measure_performance.xml
@@ -7,6 +7,7 @@
   <property name="pixelDelayFile" value="classpath:/default/delays_lightpulser_20150217.csv" />
 
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="file:src/main/resources/aux/" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
   <stream id="fact" class="fact.io.zfits.ZFitsStream"  url="${infile}"/>

diff --git a/examples/stdAnalysis/data/analysis.xml b/examples/stdAnalysis/data/analysis.xml
@@ -9,8 +9,9 @@
     <property name="pixelDelayFile" value="classpath:/default/delays_lightpulser_20150217.csv" />
 
     <property name="auxFolder" value="file:src/main/resources/aux/" />
-    <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
 
+    <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
     <!--In case you want to analyze .zfits files.-->

diff --git a/examples/stdAnalysis/mc/analysis_mc.xml b/examples/stdAnalysis/mc/analysis_mc.xml
@@ -8,7 +8,7 @@
     <property name="integralGainFile" value="classpath:/default/defaultIntegralGains.csv" />
 
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
-
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <stream id="fact" class="fact.io.FITSStream" url="${infile}"/>
 
     <process id="2" input="fact">

diff --git a/examples/stdAnalysis/mc/analysis_mc_dispRF.xml b/examples/stdAnalysis/mc/analysis_mc_dispRF.xml
@@ -11,6 +11,7 @@
     <property name="outfile" value="file:/home/fabian/testoutfile_mc.txt" />
 
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
 
     <stream id="fact" class="fact.io.FITSStream" url="${infile}"/>
 

diff --git a/examples/studies/closedShutterGainCalibration.xml b/examples/studies/closedShutterGainCalibration.xml
@@ -8,6 +8,7 @@
   <property name="pixelDelayFile" value="classpath:/default/delays_20141114.csv" />
 
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="file:src/main/resources/aux/" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
   <!-- Open Shutter Data: -->

diff --git a/examples/studies/extractionTest.xml b/examples/studies/extractionTest.xml
@@ -6,6 +6,7 @@
   <property name="integralGainFile" value="classpath:/default/defaultIntegralGains.csv" />
 
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
 
   <!-- Name of the stream and url to the input file -->
   <stream id="fact" class="fact.io.FITSStream" url="${infile}" />

diff --git a/examples/studies/fits_writer.xml b/examples/studies/fits_writer.xml
@@ -9,6 +9,7 @@
   <property name="auxFolder" value="file:src/main/resources/aux/" />
 
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
   <stream id="zfits" class="fact.io.zfits.ZFitsStream"  url="${infile}"/>

diff --git a/examples/studies/jumpStudy.xml b/examples/studies/jumpStudy.xml
@@ -6,6 +6,7 @@
   <property name="integralGainFile" value="classpath:/default/gain_sorted_20131127.csv" />
   <property name="pixelDelayFile" value="classpath:/default/delays_20141114.csv" />
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="file:src/main/resources/aux/" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <!-- Name des Streams und pfad zu dem Daten-File -->
   <!-- <stream id="fact" class="fact.io.FITSStream"  url="file:${infile}" limit="100"/> -->
   <stream id="fact" class="fact.io.FITSStream" url="${infile}" />

diff --git a/examples/studies/muon_fitting.xml b/examples/studies/muon_fitting.xml
@@ -7,6 +7,7 @@
   	<property name="drsfile" value="file:src/main/resources/testMcDrsFile.drs.fits.gz" />
 
     <property name="integralGainFile" value="classpath:/default/defaultIntegralGains.csv" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
     <!-- Name of the stream and url to the input file -->

diff --git a/examples/studies/muon_identification.xml b/examples/studies/muon_identification.xml
@@ -10,6 +10,7 @@
   <property name="auxpath" value="file:src/main/resources/aux/" />
 
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxpath}" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
   <property name="outfile" value="facttools.json" />

diff --git a/examples/studies/pedestalNsbStudy.xml b/examples/studies/pedestalNsbStudy.xml
@@ -6,6 +6,7 @@
   <property name="integralGainFile" value="classpath:/default/gain_sorted_20131127.csv" />
   <property name="pixelDelayFile" value="classpath:/default/delays_20141114.csv" />
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="file:src/main/resources/aux/" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
   <!-- Name des Streams und pfad zu dem Daten-File -->
   <stream id="fact" class="fact.io.FITSStream" url="${infile}" limit="3" />
@@ -18,7 +19,7 @@
     <fact.datacorrection.RemoveSpikes dataKey="DataCalibrated" outputKey="DataCalibrated" startCellKey="StartCellData" leftBorder="6" spikeLimit="20" topSlopeLimit="16" maxSpikeLength="4" />
     <!--<fact.filter.ArrayTimeCorrection dataKey="DataCalibrated" timeCalibConstKey="timeCalibConst" outputKey="DataCalibrated" />-->
     <fact.datacorrection.InterpolateTimeSeries calibService="calibService" interpolateTimeLine="True" dataKey="DataCalibrated" dataOutputKey="DataCalibrated" />
-    <fact.extraction.BasicExtraction dataKey="DataCalibrated" outputKeyMaxAmplPos="maxPos" outputKeyPhotonCharge="photoncharge" url="${integralGainFile}" startSearchWindow="${basicExtraction_startSearchWindow}"
+    <fact.extraction.BasicExtraction dataKey="DataCalibrated" outputKeyMaxAmplPos="maxPos" outputKeyPhotonCharge="photoncharge" gainService="gainService" startSearchWindow="${basicExtraction_startSearchWindow}"
     rangeSearchWindow="${basicExtraction_rangeSearchWindow}" rangeHalfHeightWindow="25" />
     <fact.extraction.RisingEdgeForPositions dataKey="DataCalibrated" amplitudePositionsKey="maxPos" outputKey="arrivalTimePos" maxSlopesKey="maxSlopesPos" />
     <fact.extraction.RisingEdgePolynomFit dataKey="DataCalibrated" risingEdgeKey="arrivalTimePos" outputKey="arrivalTime" numberOfPoints="11" maxSlopesKey="maxSlopes" />

diff --git a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml
@@ -11,6 +11,7 @@
     <property name="output" value="file:testoutfile.json" />
 
     <property name="auxFolder" value="file:src/main/resources/aux/" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
 
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />

diff --git a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml
@@ -11,6 +11,7 @@
     <property name="output" value="file:testoutfile.json" />
 
     <property name="auxFolder" value="file:src/main/resources/aux/" />
+    <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
     <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
 
     <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />

diff --git a/examples/studies/test_saturated_pulses.xml b/examples/studies/test_saturated_pulses.xml
@@ -4,6 +4,7 @@
   <property name="infile" value="file:src/main/resources/testMcFile.fits.gz" />
   <property name="drsfile" value="file:src/main/resources/testMcDrsFile.drs.fits.gz" />
   <property name="integralGainFile" value="classpath:/default/defaultIntegralGains.csv" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <!-- Name of the stream and url to the input file -->
   <stream id="fact" class="fact.io.FITSStream" url="${infile}" />
   <!-- Description of the process and the corresponding stream -->
@@ -19,15 +20,15 @@
     <!--badChidIds="863,868,297,927,80,873,1093,1094,527,528,721,722"-->
     <!--/>-->
     <stream.flow.Skip condition="%{data.TriggerType} != 4" />
-    <fact.extraction.BasicExtraction dataKey="DataCalibrated" outputKeyMaxAmplPos="maxPos" outputKeyPhotonCharge="photoncharge" url="${integralGainFile}" startSearchWindow="${basicExtraction_startSearchWindow}"
+    <fact.extraction.BasicExtraction dataKey="DataCalibrated" outputKeyMaxAmplPos="maxPos" outputKeyPhotonCharge="photoncharge" gainService="gainService" startSearchWindow="${basicExtraction_startSearchWindow}"
     rangeSearchWindow="${basicExtraction_rangeSearchWindow}" rangeHalfHeightWindow="25" />
     <fact.extraction.EstimateBaseline dataKey="DataCalibrated" outputKey="baseline" firstSlice="10" range="40" />
     <fact.extraction.RisingEdgeForPositions dataKey="DataCalibrated" amplitudePositionsKey="maxPos" outputKey="arrivalTimePos" maxSlopesKey="maxSlopesPos" />
     <fact.extraction.RisingEdgePolynomFit dataKey="DataCalibrated" risingEdgeKey="arrivalTimePos" outputKey="arrivalTime" numberOfPoints="11" maxSlopesKey="maxSlopes" />
     <fact.extraction.TimeOverThreshold dataKey="DataCalibrated" outputKey="timeOverThreshold" firstSliceOverThresholdOutputKey="arrivalTimeTOT" positionsKey="maxPos" threshold="1800" thresholdOutputKey="thresholdForToT" />
     <fact.extraction.TimeOverThreshold dataKey="DataCalibrated" outputKey="timeOverThreshold1000" firstSliceOverThresholdOutputKey="arrivalTimeTOT1000" positionsKey="maxPos" threshold="1000" thresholdOutputKey="thresholdForToT1000" />
     <fact.datacorrection.CorrectSaturation dataKey="DataCalibrated" outputKey="DataNoSat" totKey="timeOverThreshold" firstSliceOverThresholdKey="arrivalTimeTOT" threshold="1800" maxPosKey="maxPos" baselineKey="baseline" />
-    <fact.extraction.BasicExtraction dataKey="DataNoSat" outputKeyMaxAmplPos="maxPosNoSat" outputKeyPhotonCharge="phChargeNoSat" url="${integralGainFile}" startSearchWindow="${basicExtraction_startSearchWindow}"
+    <fact.extraction.BasicExtraction dataKey="DataNoSat" outputKeyMaxAmplPos="maxPosNoSat" outputKeyPhotonCharge="phChargeNoSat" gainService="gainService" startSearchWindow="${basicExtraction_startSearchWindow}"
     rangeSearchWindow="${basicExtraction_rangeSearchWindow}" rangeHalfHeightWindow="25" />
     <fact.extraction.PhotonChargeTimeOverThreshold timeOverThresholdKey="timeOverThreshold" thresholdKey="thresholdForToT" outputKey="photonchargeSaturated" />
     <fact.extraction.PhotonChargeTimeOverThreshold timeOverThresholdKey="timeOverThreshold1000" thresholdKey="thresholdForToT1000" outputKey="photonchargeSaturated1000" />

diff --git a/examples/studies/timeCalibrationComparison.xml b/examples/studies/timeCalibrationComparison.xml
@@ -6,6 +6,7 @@
   <property name="integralGainFile" value="classpath:/default/gain_sorted_20131127.csv" />
   <property name="pixelDelayFile" value="classpath:/default/delays_20141114.csv" />
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="file:src/main/resources/aux/" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
   <!-- Name of the stream and url to the input file -->
   <stream id="fact" class="fact.io.FITSStream" url="file:${infile}" />

diff --git a/examples/viewer.xml b/examples/viewer.xml
@@ -8,6 +8,7 @@
   <property name="auxFolder" value="file:src/main/resources/aux/" />
 
   <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
+  <service id="gainService" class="fact.calibrationservice.SinglePulseGainCalibService" integralGainFile="${integralGainFile}" />
   <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
 
   <stream id="fact" class="fact.io.hdureader.FITSStream" url="${infile}"/>
@@ -25,7 +26,7 @@
     <fact.extraction.SinglePulseExtraction
         dataKey="DataCalibrated"
         outputKey="PhotonArrivals"
-        url="${integralGainFile}"
+        gainService="gainService"
     />
 
     <fact.utils.ConvertSinglePulses2Timeseries

diff --git a/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java b/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java
@@ -0,0 +1,69 @@
+package fact.calibrationservice;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.joda.time.DateTime;
+import stream.annotations.Parameter;
+import stream.io.SourceURL;
+import stream.io.CsvStream;
+import stream.Data;
+import fact.Constants;
+
+/**
+ *
+ **/
+public class SinglePulseGainCalibService implements CalibrationService {
+
+    Logger log = LoggerFactory.getLogger(SinglePulseGainCalibService.class);
+
+    boolean isInit = false;
+    public double[] integralSinglePulseGain;
+
+    @Parameter(
+    	required = false, 
+    	description = "The path to the integral single pulse gain file."
+    )
+    SourceURL integralGainFile;
+
+	public void init() {
+        integralSinglePulseGain = new double[Constants.NUMBEROFPIXEL];
+        Data integralGainData = null;
+        try {
+            CsvStream stream = new CsvStream(integralGainFile, " ");
+            stream.setHeader(false);
+            stream.init();
+            integralGainData = stream.readNext();
+
+            for (int i = 0 ; i < Constants.NUMBEROFPIXEL ; i++){
+                String key = "column:" + (i);
+                integralSinglePulseGain[i] = (Double) integralGainData.get(key);
+            }
+
+        } catch (Exception e) {
+            log.error(
+            	"Failed to load the integral single pulse gain file: {}", 
+            	e.getMessage());
+            e.printStackTrace();
+        }
+    }
+
+	public double[] getIntegralSinglePulseGain() {
+		if (isInit == false){
+			init();
+			isInit = true;
+		}
+		return integralSinglePulseGain;
+	}
+
+    @Override
+    public void reset() throws Exception {
+    }
+
+    public void setIntegralGainFile(SourceURL integralGainFile) {
+        this.integralGainFile = integralGainFile;
+    }
+
+	public int[] getBadPixel(DateTime eventTimeStamp) {return new int[0];}
+
+	public int[] getNotUsablePixels(DateTime eventTimeStamp) {return new int[0];}
+}
diff --git a/src/main/java/fact/extraction/BasicExtraction.java b/src/main/java/fact/extraction/BasicExtraction.java
@@ -2,14 +2,13 @@
 
 import fact.Constants;
 import fact.Utils;
+import fact.calibrationservice.SinglePulseGainCalibService;
 import org.jfree.chart.plot.IntervalMarker;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import stream.Data;
 import stream.Processor;
 import stream.annotations.Parameter;
-import stream.io.CsvStream;
-import stream.io.SourceURL;
 
 /**
  * This processor performs a basic extraction on the data array. It contains three steps:
@@ -31,9 +30,8 @@ public class BasicExtraction implements Processor {
 	protected String outputKeyMaxAmplPos = null;
 	@Parameter(required = true, description="outputKey for the calculated photoncharge")
 	protected String outputKeyPhotonCharge = null;
-
-	@Parameter(required = false, description = "The url to the inputfiles for the gain calibration constants",defaultValue="file:src/main/resources/defaultIntegralGains.csv")
-	protected SourceURL url = null;
+    @Parameter(required = true, description = "The calibration service for the integral single pulse gain")
+    SinglePulseGainCalibService gainService;
 	@Parameter(required = false, description="start slice of the search window for the max amplitude", defaultValue="35")
 	protected int startSearchWindow = 35;
 	@Parameter(required = false, description="range of the search window for the max amplitude", defaultValue="90")
@@ -55,6 +53,7 @@ public Data process(Data input) {
 
 		int roi = (Integer) input.get("NROI");
 		npix = (Integer) input.get("NPIX");
+		integralGains = gainService.getIntegralSinglePulseGain();
 
 		double[] data = (double[]) input.get(dataKey);
 
@@ -133,29 +132,6 @@ public double calculateIntegral(int px, int startingPosition, int integralSize,
 		}
 		return integral;
 	}
-
-
-	public double[] loadIntegralGainFile(SourceURL inputUrl, Logger log) {
-		double[] integralGains = new double[npix];
-		Data integralGainData = null;
-		try {
-			CsvStream stream = new CsvStream(inputUrl, " ");
-			stream.setHeader(false);
-			stream.init();
-			integralGainData = stream.readNext();
-
-			for (int i = 0 ; i < npix ; i++){
-				String key = "column:" + (i);
-				integralGains[i] = (Double) integralGainData.get(key);
-			}
-			return integralGains;
-
-		} catch (Exception e) {
-			log.error("Failed to load integral Gain data: {}", e.getMessage());
-			e.printStackTrace();
-			return null;
-		}
-	}
 
 	public String getDataKey() {
 		return dataKey;
@@ -165,6 +141,10 @@ public void setDataKey(String dataKey) {
 		this.dataKey = dataKey;
 	}
 
+    public void setGainService(SinglePulseGainCalibService gainService) {
+        this.gainService = gainService;
+    }
+
 	public String getOutputKeyMaxAmplPos() {
 		return outputKeyMaxAmplPos;
 	}
@@ -221,19 +201,4 @@ public int getValidMinimalSlice() {
 	public void setValidMinimalSlice(int validMinimalSlice) {
 		this.validMinimalSlice = validMinimalSlice;
 	}
-
-	public void setUrl(SourceURL url) {
-		try {
-			integralGains = loadIntegralGainFile(url,log);
-		} catch (Exception e) {
-			throw new RuntimeException(e.getMessage());
-		}
-		this.url = url;
-	}
-
-	public SourceURL getUrl() {
-		return url;
-	}
-
-
 }