updates for timing (#211)

.github/workflows/tests.yml

@@ -338,6 +338,13 @@ jobs:
         - 5672:5672
         - 15672:15672
     steps:
+      - uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: 'true'
+          large-packages: false  # TODO: remove once https://github.com/jlumbroso/free-disk-space/issues/4 is fixed
+      - uses: actionhippie/swap-space@v1
+        with:
+          size: 10G
       - uses: actions/checkout@v3
       - uses: docker/setup-buildx-action@v2
       - uses: docker/build-push-action@v3
@@ -359,18 +366,8 @@ jobs:
           cd ./resources/launch_scripts
           ./local-scan.sh
 
-      - name: test output against known result (.npz/.json)
+      - name: test output against known result (.json)
         run: |
-          if [[ ${{ matrix.reco_algo }} == 'millipede_wilks' ]]; then
-            if [[ $(date +"%s") -gt 1695235370 ]]; then
-              echo "Date and time (GMT): Wednesday, September 20, 2023 18:42:50"
-              echo "tests should pass nowish, ping Tianlu about the ftp-v1 timing spline table fix"
-              exit 1
-            else
-              exit 0
-            fi
-          fi
-
           ls $SKYSCAN_OUTPUT_DIR
           # get newest run*.json
           outfile=$(find $SKYSCAN_OUTPUT_DIR -type f -name "run*.json" -exec stat -c '%y %n'  {} + | sort | tail -1 | awk '{print $4}')
@@ -531,4 +528,4 @@ jobs:
         git_committer_email: ${{ env.BOT_EMAIL }}
         github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
         # repository_username: __token__
-        # repository_password: ${{ secrets.PYPI_TOKEN }}
+        # repository_password: ${{ secrets.PYPI_TOKEN }}

dependencies-from-Dockerfile.log

@@ -26,13 +26,13 @@ decorator==4.4.2
 defusedxml==0.7.1
 docutils==0.20.1
 et-xmlfile==1.0.1
-ewms-pilot==0.12.0
+ewms-pilot==0.12.2
 fonttools==4.29.1
 fs==2.4.12
 gast==0.5.2
 healpy==1.15.0
 htchirp==2.0
-htcondor==10.7.0
+htcondor==10.8.0
 html5lib==1.1
 humanfriendly==10.0
 hypothesis==6.36.0
@@ -122,7 +122,7 @@ tables==3.7.0
 toml==0.10.2
 tornado==6.3.3
 traitlets==5.1.1
-typing_extensions==4.7.1
+typing_extensions==4.8.0
 ufoLib2==0.13.1
 unicodedata2==14.0.0
 UNKNOWN @ file:///root/nuflux-2.0.4

dependencies-from-Dockerfile_pulsar.log

@@ -26,7 +26,7 @@ decorator==4.4.2
 defusedxml==0.7.1
 docutils==0.20.1
 et-xmlfile==1.0.1
-ewms-pilot==0.12.0
+ewms-pilot==0.12.2
 fonttools==4.29.1
 fs==2.4.12
 gast==0.5.2
@@ -122,7 +122,7 @@ tables==3.7.0
 toml==0.10.2
 tornado==6.3.3
 traitlets==5.1.1
-typing_extensions==4.7.1
+typing_extensions==4.8.0
 ufoLib2==0.13.1
 unicodedata2==14.0.0
 UNKNOWN @ file:///root/nuflux-2.0.4

setup.cfg

@@ -18,7 +18,7 @@ branch = main
 
 [options]  # generated by wipac:cicd_setup_builder: python_requires, packages
 install_requires =
-	ewms-pilot==0.12.0 # the extras here are the same as 'oms-mqclient', so just define one place
+	ewms-pilot==0.12.2 # the extras here are the same as 'oms-mqclient', so just define one place
 	healpy
 	icecube-skyreader
 	iminuit

skymap_scanner/client/reco_icetray.py

@@ -11,7 +11,7 @@
 from pathlib import Path
 from typing import Any, List, Union
 
-from I3Tray import I3Tray  # type: ignore[import]
+from icecube.icetray import I3Tray  # type: ignore[import]
 from icecube import (  # type: ignore[import]  # noqa: F401
     dataio,
     frame_object_diff,

skymap_scanner/recos/common/pulse_proc.py

@@ -1,8 +1,8 @@
 import copy
 import numpy
-from typing import Final, Union
+from typing import Final, Union, List
 
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import dataclasses  # type: ignore[import]
 
 
@@ -77,3 +77,50 @@ def late_pulse_cleaning(
     frame[output_pulses_name + "TimeRange"] = copy.deepcopy(
         frame[orig_pulses_name + "TimeRange"]
     )
+
+
+def pulse_cleaning(
+    frame,
+    input_pulses_name: str,
+    output_pulses_name: str,
+    residual,
+):
+    pulses = dataclasses.I3RecoPulseSeriesMap.from_frame(frame, input_pulses_name)
+    mask = dataclasses.I3RecoPulseSeriesMapMask(frame, input_pulses_name)
+    counter, charge = 0, 0
+    qtot = 0
+    times = dataclasses.I3TimeWindowSeriesMap()
+    all_ts: List[float] = []
+    all_cs: List[float] = []
+    for omkey, ps in pulses.items():
+        ts = numpy.asarray([p.time for p in ps])
+        all_ts.extend(ts)
+        cs = numpy.asarray([p.charge for p in ps])
+        all_cs.extend(cs)
+    tw_start = weighted_quantile(numpy.asarray(all_ts), numpy.asarray(all_cs), 0.1) - 1000
+    tw_stop = weighted_quantile(numpy.asarray(all_ts), numpy.asarray(all_cs), 0.95) + 1000
+    for omkey, ps in pulses.items():
+        ts = numpy.asarray([p.time for p in ps])
+        cs = numpy.asarray([p.charge for p in ps])
+        median = weighted_median(ts, cs)
+        dts = numpy.ediff1d(ts)
+        median_dts = numpy.median(dts)
+        qtot += cs.sum()
+        for p in ps:
+            if median_dts > 1200 and len(dts) > 1:
+                # attempt to mask out correlated noise
+                mask.set(omkey, p, False)
+            elif p.time >= (latest_time := min(median + residual, tw_stop)) or p.time < tw_start:
+                if omkey not in times:
+                    tws = dataclasses.I3TimeWindowSeries()
+                    tws.append(
+                        dataclasses.I3TimeWindow(latest_time, numpy.inf)
+                        )  # this defines the **excluded** time window
+                    times[omkey] = tws
+                mask.set(omkey, p, False)
+                counter += 1
+                charge += p.charge
+
+    frame[output_pulses_name] = mask
+    frame[output_pulses_name + "TimeWindows"] = times
+    frame[output_pulses_name + "TimeRange"] = dataclasses.I3TimeWindow(tw_start, tw_stop)

skymap_scanner/recos/common/vertex_gen.py

@@ -3,7 +3,7 @@
 import numpy as np
 
 from icecube import dataclasses  # type: ignore[import]
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 
 
 class VertexGenerator:

skymap_scanner/recos/dummy.py

@@ -6,7 +6,7 @@
 import time
 from typing import List, Final
 
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import (  # type: ignore[import]  # noqa: F401
     dataclasses,
     frame_object_diff,

skymap_scanner/recos/millipede_original.py

@@ -10,7 +10,7 @@
 
 import numpy
 
-from I3Tray import I3Units
+from icecube.icetray import I3Units
 from icecube import (  # noqa: F401
     VHESelfVeto,
     dataclasses,

skymap_scanner/recos/millipede_wilks.py

@@ -10,7 +10,7 @@
 from typing import Final, List, Tuple
 
 import numpy
-from I3Tray import I3Units
+from icecube.icetray import I3Units
 from icecube import (  # noqa: F401
     VHESelfVeto,
     dataclasses,
@@ -22,14 +22,14 @@
     millipede,
     photonics_service,
     recclasses,
-    simclasses,
+    simclasses
 )
 from icecube.icetray import I3Frame
 
 from .. import config as cfg
 from ..utils.pixel_classes import RecoPixelVariation
 from . import RecoInterface, VertexGenerator
-from .common.pulse_proc import mask_deepcore, late_pulse_cleaning
+from .common.pulse_proc import mask_deepcore, pulse_cleaning
 
 class MillipedeWilks(RecoInterface):
     """Reco logic for millipede."""
@@ -38,11 +38,13 @@ class MillipedeWilks(RecoInterface):
     FTP_ABS_SPLINE = "cascade_single_spice_ftp-v1_flat_z20_a5.abs.fits"
     FTP_PROB_SPLINE = "cascade_single_spice_ftp-v1_flat_z20_a5.prob.fits"
     FTP_EFFD_SPLINE = "cascade_effectivedistance_spice_ftp-v1_z20.eff.fits"
+    FTP_EFFP_SPLINE = "cascade_effectivedistance_spice_ftp-v1_z20.prob.fits"
+    FTP_TMOD_SPLINE = "cascade_effectivedistance_spice_ftp-v1_z20.tmod.fits"
 
-    SPLINE_REQUIREMENTS = [FTP_ABS_SPLINE, FTP_PROB_SPLINE, FTP_EFFD_SPLINE]
+    SPLINE_REQUIREMENTS = [FTP_ABS_SPLINE, FTP_PROB_SPLINE, FTP_EFFD_SPLINE,
+                           FTP_EFFP_SPLINE, FTP_TMOD_SPLINE]
     # Constants ########################################################
 
-    pulsesName_orig = cfg.INPUT_PULSES_NAME
     pulsesName = cfg.INPUT_PULSES_NAME + "IC"
     pulsesName_cleaned = pulsesName+'LatePulseCleaned'
 
@@ -65,12 +67,16 @@ def setup_reco(self):
         abs_spline: str = datastager.get_filepath(self.FTP_ABS_SPLINE)
         prob_spline: str = datastager.get_filepath(self.FTP_PROB_SPLINE)
         effd_spline: str = datastager.get_filepath(self.FTP_EFFD_SPLINE)
+        effp_spline: str = datastager.get_filepath(self.FTP_EFFP_SPLINE)
+        tmod_spline: str = datastager.get_filepath(self.FTP_TMOD_SPLINE)
 
         self.cascade_service = photonics_service.I3PhotoSplineService(
             abs_spline, prob_spline, timingSigma=0.0,
             effectivedistancetable = effd_spline,
             tiltTableDir = os.path.expandvars('$I3_BUILD/ice-models/resources/models/ICEMODEL/spice_ftp-v1/'),
-            quantileEpsilon=1
+            quantileEpsilon=1,
+            effectivedistancetableprob = effp_spline,
+            effectivedistancetabletmod = tmod_spline
         )
 
         self.muon_service = None
@@ -172,17 +178,17 @@ def skipunhits(frame, output, pulses):
 
             frame[output] = unhits
 
-        tray.Add(skipunhits, output='OtherUnhits', pulses=cls.pulsesName)
-        ExcludedDOMs.append('OtherUnhits')
-
         ##################
-
-        tray.AddModule(late_pulse_cleaning, "LatePulseCleaning",
+        tray.AddModule(pulse_cleaning, "LatePulseCleaning",
                        input_pulses_name=cls.pulsesName,
                        output_pulses_name=cls.pulsesName_cleaned,
-                       orig_pulses_name=cls.pulsesName_orig,
                        residual=1.5e3*I3Units.ns)
-        return ExcludedDOMs + [cls.pulsesName_cleaned+'TimeWindows']
+        ExcludedDOMs.append(cls.pulsesName_cleaned+'TimeWindows')
+
+        tray.Add(skipunhits, output='OtherUnhits', pulses=cls.pulsesName_cleaned)
+        ExcludedDOMs.append('OtherUnhits')
+        return ExcludedDOMs
+
 
     @icetray.traysegment
     def traysegment(self, tray, name, logger, seed=None):
@@ -207,7 +213,7 @@ def notify0(frame):
             Pulses=self.pulsesName_cleaned,
             BinSigma=2,
             MinTimeWidth=25,
-            RelUncertainty=0.3)
+            RelUncertainty=1)
 
         tray.AddService('I3GSLRandomServiceFactory','I3RandomService')
 

skymap_scanner/recos/splinempe.py

@@ -6,7 +6,7 @@
 from typing import Final, List
 
 
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 
 # NOTE: icecube module imports are required to make IceTray modules and services available.
 from icecube import (  # type: ignore[import]  # noqa: F401

skymap_scanner/server/start_scan.py

@@ -15,7 +15,7 @@
 import healpy  # type: ignore[import]
 import mqclient as mq
 import numpy
-from I3Tray import I3Units  # type: ignore[import]
+from icecube.icetray import I3Units  # type: ignore[import]
 from icecube import (  # type: ignore[import]
     astro,
     dataclasses,

skymap_scanner/utils/prepare_frames.py

@@ -9,7 +9,7 @@
 import os
 from typing import Union, List
 
-from I3Tray import I3Tray  # type: ignore[import]
+from icecube.icetray import I3Tray  # type: ignore[import]
 from icecube import icetray  # type: ignore[import]
 from icecube.frame_object_diff.segments import uncompress  # type: ignore[import]
 

tests/compare_scan_results.py

@@ -8,7 +8,7 @@
 from skyreader import SkyScanResult
 from wipac_dev_tools import logging_tools
 
-RTOL_PER_FIELD = {"llh": 0.18, "E_in": 0.7, "E_tot": 0.7}
+RTOL_PER_FIELD = {"llh": 0.15, "E_in": 0.1, "E_tot": 0.1}
 
 
 def read_file(filepath: Path) -> SkyScanResult: