updates for timing

.github/workflows/tests.yml

@@ -359,18 +359,8 @@ jobs:
           cd ./resources/launch_scripts
           ./local-scan.sh
 
-      - name: test output against known result (.npz/.json)
+      - name: test output against known result (.json)
         run: |
-          if [[ ${{ matrix.reco_algo }} == 'millipede_wilks' ]]; then
-            if [[ $(date +"%s") -gt 1695235370 ]]; then
-              echo "Date and time (GMT): Wednesday, September 20, 2023 18:42:50"
-              echo "tests should pass nowish, ping Tianlu about the ftp-v1 timing spline table fix"
-              exit 1
-            else
-              exit 0
-            fi
-          fi
-
           ls $SKYSCAN_OUTPUT_DIR
           # get newest run*.json
           outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n'  {} + | sort | tail -1 | awk '{print $4}')
@@ -531,4 +521,4 @@ jobs:
         git_committer_email: ${{ env.BOT_EMAIL }}
         github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
         # repository_username: __token__
-        # repository_password: ${{ secrets.PYPI_TOKEN }}
+        # repository_password: ${{ secrets.PYPI_TOKEN }}

skymap_scanner/client/reco_icetray.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 from typing import Any, List, Union
 
-from I3Tray import I3Tray  # type: ignore[import]
+from icecube.icetray import I3Tray  # type: ignore[import]
 from icecube import (  # type: ignore[import]  # noqa: F401
     dataio,
     frame_object_diff,

skymap_scanner/recos/common/pulse_proc.py

@@ -1,8 +1,8 @@
 import copy
 import numpy
-from typing import Final, Union
+from typing import Final, Union, List
 
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import dataclasses  # type: ignore[import]
 
 
@@ -77,3 +77,50 @@ def late_pulse_cleaning(
     frame[output_pulses_name + "TimeRange"] = copy.deepcopy(
         frame[orig_pulses_name + "TimeRange"]
     )
+
+
+def pulse_cleaning(
+    frame,
+    input_pulses_name: str,
+    output_pulses_name: str,
+    residual,
+):
+    pulses = dataclasses.I3RecoPulseSeriesMap.from_frame(frame, input_pulses_name)
+    mask = dataclasses.I3RecoPulseSeriesMapMask(frame, input_pulses_name)
+    counter, charge = 0, 0
+    qtot = 0
+    times = dataclasses.I3TimeWindowSeriesMap()
+    all_ts: List[float] = []
+    all_cs: List[float] = []
+    for omkey, ps in pulses.items():
+        ts = numpy.asarray([p.time for p in ps])
+        all_ts.extend(ts)
+        cs = numpy.asarray([p.charge for p in ps])
+        all_cs.extend(cs)
+    tw_start = weighted_quantile(numpy.asarray(all_ts), numpy.asarray(all_cs), 0.1) - 1000
+    tw_stop = weighted_quantile(numpy.asarray(all_ts), numpy.asarray(all_cs), 0.95) + 1000
+    for omkey, ps in pulses.items():
+        ts = numpy.asarray([p.time for p in ps])
+        cs = numpy.asarray([p.charge for p in ps])
+        median = weighted_median(ts, cs)
+        dts = numpy.ediff1d(ts)
+        median_dts = numpy.median(dts)
+        qtot += cs.sum()
+        for p in ps:
+            if median_dts > 1200 and len(dts) > 1:
+                # attempt to mask out correlated noise
+                mask.set(omkey, p, False)
+            elif p.time >= (latest_time := min(median + residual, tw_stop)) or p.time < tw_start:
+                if omkey not in times:
+                    tws = dataclasses.I3TimeWindowSeries()
+                    tws.append(
+                        dataclasses.I3TimeWindow(latest_time, numpy.inf)
+                        )  # this defines the **excluded** time window
+                    times[omkey] = tws
+                mask.set(omkey, p, False)
+                counter += 1
+                charge += p.charge
+
+    frame[output_pulses_name] = mask
+    frame[output_pulses_name + "TimeWindows"] = times
+    frame[output_pulses_name + "TimeRange"] = dataclasses.I3TimeWindow(tw_start, tw_stop)

skymap_scanner/recos/common/vertex_gen.py

@@ -3,7 +3,7 @@
 import numpy as np
 
 from icecube import dataclasses  # type: ignore[import]
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 
 
 class VertexGenerator:

skymap_scanner/recos/dummy.py

@@ -6,7 +6,7 @@
 import time
 from typing import List, Final
 
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import (  # type: ignore[import]  # noqa: F401
     dataclasses,
     frame_object_diff,

skymap_scanner/recos/millipede_original.py

@@ -10,7 +10,7 @@
 
 import numpy
 
-from I3Tray import I3Units
+from icecube.icetray import I3Units
 from icecube import (  # noqa: F401
     VHESelfVeto,
     dataclasses,

skymap_scanner/recos/millipede_wilks.py

@@ -10,7 +10,7 @@
 from typing import Final, List, Tuple
 
 import numpy
-from I3Tray import I3Units
+from icecube.icetray import I3Units
 from icecube import (  # noqa: F401
     VHESelfVeto,
     dataclasses,
@@ -22,14 +22,14 @@
     millipede,
     photonics_service,
     recclasses,
-    simclasses,
+    simclasses
 )
 from icecube.icetray import I3Frame
 
 from .. import config as cfg
 from ..utils.pixel_classes import RecoPixelVariation
 from . import RecoInterface, VertexGenerator
-from .common.pulse_proc import mask_deepcore, late_pulse_cleaning
+from .common.pulse_proc import mask_deepcore, pulse_cleaning
 
 class MillipedeWilks(RecoInterface):
     """Reco logic for millipede."""
@@ -38,11 +38,13 @@ class MillipedeWilks(RecoInterface):
     FTP_ABS_SPLINE = "cascade_single_spice_ftp-v1_flat_z20_a5.abs.fits"
     FTP_PROB_SPLINE = "cascade_single_spice_ftp-v1_flat_z20_a5.prob.fits"
     FTP_EFFD_SPLINE = "cascade_effectivedistance_spice_ftp-v1_z20.eff.fits"
+    FTP_EFFP_SPLINE = "cascade_effectivedistance_spice_ftp-v1_z20.prob.fits"
+    FTP_TMOD_SPLINE = "cascade_effectivedistance_spice_ftp-v1_z20.tmod.fits"
 
-    SPLINE_REQUIREMENTS = [FTP_ABS_SPLINE, FTP_PROB_SPLINE, FTP_EFFD_SPLINE]
+    SPLINE_REQUIREMENTS = [FTP_ABS_SPLINE, FTP_PROB_SPLINE, FTP_EFFD_SPLINE,
+                           FTP_EFFP_SPLINE, FTP_TMOD_SPLINE]
     # Constants ########################################################
 
-    pulsesName_orig = cfg.INPUT_PULSES_NAME
     pulsesName = cfg.INPUT_PULSES_NAME + "IC"
     pulsesName_cleaned = pulsesName+'LatePulseCleaned'
 
@@ -65,12 +67,16 @@ def setup_reco(self):
         abs_spline: str = datastager.get_filepath(self.FTP_ABS_SPLINE)
         prob_spline: str = datastager.get_filepath(self.FTP_PROB_SPLINE)
         effd_spline: str = datastager.get_filepath(self.FTP_EFFD_SPLINE)
+        effp_spline: str = datastager.get_filepath(self.FTP_EFFP_SPLINE)
+        tmod_spline: str = datastager.get_filepath(self.FTP_TMOD_SPLINE)
 
         self.cascade_service = photonics_service.I3PhotoSplineService(
             abs_spline, prob_spline, timingSigma=0.0,
             effectivedistancetable = effd_spline,
             tiltTableDir = os.path.expandvars('$I3_BUILD/ice-models/resources/models/ICEMODEL/spice_ftp-v1/'),
-            quantileEpsilon=1
+            quantileEpsilon=1,
+            effectivedistancetableprob = effp_spline,
+            effectivedistancetabletmod = tmod_spline
         )
 
         self.muon_service = None
@@ -172,17 +178,17 @@ def skipunhits(frame, output, pulses):
 
             frame[output] = unhits
 
-        tray.Add(skipunhits, output='OtherUnhits', pulses=cls.pulsesName)
-        ExcludedDOMs.append('OtherUnhits')
-
         ##################
-
-        tray.AddModule(late_pulse_cleaning, "LatePulseCleaning",
+        tray.AddModule(pulse_cleaning, "LatePulseCleaning",
                        input_pulses_name=cls.pulsesName,
                        output_pulses_name=cls.pulsesName_cleaned,
-                       orig_pulses_name=cls.pulsesName_orig,
                        residual=1.5e3*I3Units.ns)
-        return ExcludedDOMs + [cls.pulsesName_cleaned+'TimeWindows']
+        ExcludedDOMs.append(cls.pulsesName_cleaned+'TimeWindows')
+
+        tray.Add(skipunhits, output='OtherUnhits', pulses=cls.pulsesName_cleaned)
+        ExcludedDOMs.append('OtherUnhits')
+        return ExcludedDOMs
+
 
     @icetray.traysegment
     def traysegment(self, tray, name, logger, seed=None):
@@ -207,7 +213,7 @@ def notify0(frame):
             Pulses=self.pulsesName_cleaned,
             BinSigma=2,
             MinTimeWidth=25,
-            RelUncertainty=0.3)
+            RelUncertainty=1)
 
         tray.AddService('I3GSLRandomServiceFactory','I3RandomService')
 

skymap_scanner/recos/splinempe.py

@@ -6,7 +6,7 @@
 from typing import Final, List
 
 
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 
 # NOTE: icecube module imports are required to make IceTray modules and services available.
 from icecube import (  # type: ignore[import]  # noqa: F401

skymap_scanner/server/start_scan.py

@@ -15,7 +15,7 @@
 import healpy  # type: ignore[import]
 import mqclient as mq
 import numpy
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import (  # type: ignore[import]
     astro,
     dataclasses,

skymap_scanner/utils/prepare_frames.py

@@ -9,7 +9,7 @@
 import os
 from typing import Union, List
 
-from I3Tray import I3Tray  # type: ignore[import]
+from icecube.icetray import I3Tray  # type: ignore[import]
 from icecube import icetray  # type: ignore[import]
 from icecube.frame_object_diff.segments import uncompress  # type: ignore[import]
 

tests/compare_scan_results.py

@@ -8,7 +8,7 @@
 from skyreader import SkyScanResult
 from wipac_dev_tools import logging_tools
 
-RTOL_PER_FIELD = {"llh": 0.18, "E_in": 0.7, "E_tot": 0.7}
+RTOL_PER_FIELD = {"llh": 0.15, "E_in": 0.1, "E_tot": 0.1}
 
 
 def read_file(filepath: Path) -> SkyScanResult:

tests/data/results_json/millipede_wilks/run00127907.evt000020178442.HESE_1.json

@@ -7,77 +7,77 @@
             "E_tot"
         ],
         "data": [
-            [
+          [
                 0,
-                333.8115059639348,
-                8563.892342009616,
-                8563.892342009616
+                126.38114632411437,
+                5241.262194364837,
+                5241.262194364837
             ],
             [
                 1,
-                333.7291416388994,
-                14753.6997025439,
-                14753.6997025439
+                125.18614261171011,
+                4770.901739386568,
+                97317068489.0751
             ],
             [
                 2,
-                324.54558625576686,
-                7264.871600857262,
-                10515.68731299288
+                120.5334856322618,
+                4541.573130514726,
+                4541.573130514726
             ],
             [
                 3,
-                334.7846753468303,
-                6598.2632814149565,
-                7577.056152015813
+                127.51615963455372,
+                5157.642800402355,
+                5201.420713265329
             ],
             [
                 4,
-                327.8656127148552,
-                7229.275061582002,
-                14185.14125705467
+                120.58305726767188,
+                4179.421962017046,
+                17697.095837204422
             ],
             [
                 5,
-                325.29146978913224,
-                9977.181123899314,
-                7895763.342219342
+                125.55386417676894,
+                5401.557410295537,
+                5401.557410295537
             ],
             [
                 6,
-                331.5395815324554,
-                7454.208046150723,
-                50303.60385834099
+                120.81547847480626,
+                4470.270273289149,
+                14979.397763776346
             ],
             [
                 7,
-                324.70916170926915,
-                11017.804391750058,
-                18576754.509056866
+                124.2783280347043,
+                5712.5845051136175,
+                5712.5845051136175
             ],
             [
                 8,
-                311.3694736493094,
-                8137.554624196082,
-                9797.958766249872
+                116.42702192317712,
+                3800.472255088158,
+                3800.472255088158
             ],
             [
                 9,
-                324.84472870443784,
-                6282.414244617621,
-                964695010.097
+                125.83301174352532,
+                4415.348401578086,
+                4415.348401578086
             ],
             [
                 10,
-                329.011384345754,
-                7206.515271762042,
-                7206.515271762042
+                124.16439913624171,
+                4295.2972095867735,
+                4295.2972095867735
             ],
             [
                 11,
-                323.2556751592042,
-                354786.45979516796,
-                354786.45979516796
+                120.79128966342194,
+                3607.4755539002545,
+                4978041313.265813
             ]
         ],
         "metadata": {
@@ -89,4 +89,4 @@
             "is_real_event": false
         }
     }
-}
+}