From 58e1b5fb8e21d3e569d38418f8af516e4344470f Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Tue, 19 Dec 2023 09:36:42 +0100 Subject: [PATCH 01/10] avoid TimeTier crash --- snews_pt/messages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index eededbc..351ceca 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -315,9 +315,10 @@ class SNEWSTimingTierMessage(SNEWSMessage): def __init__(self, p_val=None, timing_series=None, **kwargs): super().__init__(self.fields, p_val=p_val, - timing_series=[self.clean_time_input(t) for t in timing_series], + timing_series=timing_series, #[self.clean_time_input(t) for t in timing_series], **kwargs) + # TODO: the timing series time check should be flexible to allow for floating point seconds from the initial time def is_valid(self): """Check that parameter values are valid for this tier.""" for time in self.message_data['timing_series']: From d41900ae58e2c59d3dd9c0965533d0143cec7ef5 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Tue, 19 Dec 2023 09:37:02 +0100 Subject: [PATCH 02/10] avoid TimeTier crash --- snews_pt/messages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index 351ceca..4f81dca 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -6,6 +6,7 @@ import json import numpy as np from abc import ABC, abstractmethod +im from datetime import datetime try: From 6555b3e01439098a6f056a7ebdf1e1a8af064673 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Tue, 19 Dec 2023 09:38:04 +0100 Subject: [PATCH 03/10] avoid TimeTier crash --- snews_pt/messages.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index 4f81dca..cd5573b 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -6,8 +6,6 @@ import json import numpy as np from abc import ABC, abstractmethod -im - from datetime import datetime try: fromisoformat = datetime.fromisoformat From 20ccc79c8d97781e569706e12db386699f3e2b8f Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Mon, 8 Apr 2024 09:40:00 +0200 Subject: [PATCH 04/10] CLI fix for publish and hb --- snews_pt/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snews_pt/__main__.py b/snews_pt/__main__.py index e92d98b..45dd242 100644 --- a/snews_pt/__main__.py +++ b/snews_pt/__main__.py @@ -49,7 +49,7 @@ def publish(ctx, file, firedrill): click.clear() for f in file: if f.endswith('.json'): - SNEWSMessageBuilder.from_json(jsonfile=f, env_file=ctx.obj['env'], firedrill_mode=firedrill).send_messages() + SNEWSMessageBuilder.from_json(jsonfile=f, env_file=ctx.obj['env']).send_messages(firedrill_mode=firedrill) else: # maybe just print instead of raising @@ -69,8 +69,8 @@ def heartbeat(ctx, status, time, firedrill): message = SNEWSMessageBuilder(detector_name=ctx.obj['DETECTOR_NAME'], machine_time=time, detector_status=status, - firedrill_mode=firedrill) - message.send_messages() + ) + message.send_messages(firedrill_mode=firedrill) @main.command() From 139a0eb95441e2fd4eb8a393aa812ee4b12e18b6 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Mon, 8 Apr 2024 10:40:00 +0200 Subject: [PATCH 05/10] time tier accepts floats --- snews_pt/messages.py | 52 +++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index cd5573b..6210124 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -314,29 +314,41 @@ class SNEWSTimingTierMessage(SNEWSMessage): def __init__(self, p_val=None, timing_series=None, **kwargs): super().__init__(self.fields, p_val=p_val, - timing_series=timing_series, #[self.clean_time_input(t) for t in timing_series], + timing_series=timing_series, **kwargs) - # TODO: the timing series time check should be flexible to allow for floating point seconds from the initial time def is_valid(self): - """Check that parameter values are valid for this tier.""" - for time in self.message_data['timing_series']: - if isinstance(time, str): - time = fromisoformat(time) - time = time.isoformat() - if not self.is_test: - # time format is corrected at the base class, check if reasonable - timeobj = fromisoformat(time) - duration = (timeobj - datetime.utcnow()).total_seconds() - if (duration <= -172800.0) or (duration > 0.0): - raise ValueError(f'{self.__class__.__name__} neutrino_time must be within 48 hours of now.') - - # p_val must be a float between 0 and 1 - pv = self.message_data['p_val'] - if isinstance(pv, str): - pv = float(pv) - if not (0.0 <= pv <= 1.0): - raise ValueError(f'{self.__class__.__name__} p_value of the detection must be between 0 and 1.') + """Check that parameter values are valid for this tier. + timing series can either be a list of iso-convertible strings or a list of floats.""" + if all([isinstance(t, str) for t in self.message_data['timing_series']]): + # convert to numpy datetime objects + times_obj = np.array([np.datetime64(t) for t in self.message_data['timing_series']]).astype('datetime64[ns]') + times_obj = np.sort(times_obj) + relative_times = (times_obj - times_obj[0]).astype('timedelta64[ns]') # make sure they are always ns precision + elif all([isinstance(t, (int, float)) for t in self.message_data['timing_series']]): + # if they are relative, expect an initial neutrino time + if "neutrino_time" not in self.message_data: + raise ValueError(f'{self.__class__.__name__} neutrino_time must be provided if timing_series are relative.') + # then we assume they are relative times from the first neutrino time with ns precision + relative_times = np.array(self.message_data['timing_series']) + else: + raise ValueError(f'{self.__class__.__name__} timing_series must be a list of isoformat strings or ' + f'ns-precision floats from the first neutrino time.') + + if not self.is_test: + # Check timing validity + # expect to see a monotonic increase in the relative times + is_monotonic = np.all(np.diff(relative_times) >= 0) + if not is_monotonic: + raise ValueError(f'{self.__class__.__name__} timing_series must be in increasing order. ' + f'They represent the time after initial neutrino time') + + # p_val must be a float between 0 and 1 + pv = self.message_data['p_val'] + if isinstance(pv, str): + pv = float(pv) + if not (0.0 <= pv <= 1.0): + raise ValueError(f'{self.__class__.__name__} p_value of the detection must be between 0 and 1.') return True From ee7f06274d4db60a99a29229d39a70e7dc5dcf40 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Mon, 8 Apr 2024 13:26:29 +0200 Subject: [PATCH 06/10] revert firedrill fix --- snews_pt/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snews_pt/__main__.py b/snews_pt/__main__.py index 45dd242..e92d98b 100644 --- a/snews_pt/__main__.py +++ b/snews_pt/__main__.py @@ -49,7 +49,7 @@ def publish(ctx, file, firedrill): click.clear() for f in file: if f.endswith('.json'): - SNEWSMessageBuilder.from_json(jsonfile=f, env_file=ctx.obj['env']).send_messages(firedrill_mode=firedrill) + SNEWSMessageBuilder.from_json(jsonfile=f, env_file=ctx.obj['env'], firedrill_mode=firedrill).send_messages() else: # maybe just print instead of raising @@ -69,8 +69,8 @@ def heartbeat(ctx, status, time, firedrill): message = SNEWSMessageBuilder(detector_name=ctx.obj['DETECTOR_NAME'], machine_time=time, detector_status=status, - ) - message.send_messages(firedrill_mode=firedrill) + firedrill_mode=firedrill) + message.send_messages() @main.command() From de50ab57a00375c1e542307ca25bb789289717e9 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Mon, 8 Apr 2024 14:30:10 +0200 Subject: [PATCH 07/10] adjust floating implementation --- snews_pt/messages.py | 29 ++++++++++++++--------- snews_pt/test/test_timing_tier.py | 38 ++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index b008847..d3a98ee 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -313,33 +313,40 @@ class SNEWSTimingTierMessage(SNEWSMessage): fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val', 'is_test' ] def __init__(self, p_val=None, timing_series=None, **kwargs): + # first convert the timing series into relative times + times = self.convert_times(timing_series) super().__init__(self.fields, p_val=p_val, - timing_series=timing_series, + timing_series=times, **kwargs) - def is_valid(self): - """Check that parameter values are valid for this tier. - timing series can either be a list of iso-convertible strings or a list of floats.""" - if all([isinstance(t, str) for t in self.message_data['timing_series']]): + def convert_times(self, timing_series): + if all([isinstance(t, str) for t in timing_series]): # convert to numpy datetime objects - times_obj = np.array([np.datetime64(t) for t in self.message_data['timing_series']]).astype('datetime64[ns]') + times_obj = np.array([np.datetime64(t) for t in timing_series]).astype('datetime64[ns]') times_obj = np.sort(times_obj) relative_times = (times_obj - times_obj[0]).astype('timedelta64[ns]') # make sure they are always ns precision - elif all([isinstance(t, (int, float)) for t in self.message_data['timing_series']]): + relative_times = [int(t//np.timedelta64(1, 'ns')) for t in relative_times] + elif all([isinstance(t, (int, float)) for t in timing_series]): # if they are relative, expect an initial neutrino time - if "neutrino_time" not in self.message_data: - raise ValueError(f'{self.__class__.__name__} neutrino_time must be provided if timing_series are relative.') + # if "neutrino_time" not in self.meta: + # raise ValueError(f'{self.__class__.__name__} neutrino_time must be provided if timing_series are relative.') # then we assume they are relative times from the first neutrino time with ns precision - relative_times = np.array(self.message_data['timing_series']) + relative_times = timing_series if isinstance(timing_series, list) else list(timing_series) else: raise ValueError(f'{self.__class__.__name__} timing_series must be a list of isoformat strings or ' f'ns-precision floats from the first neutrino time.') + return relative_times + + + def is_valid(self): + """Check that parameter values are valid for this tier. + timing series can either be a list of iso-convertible strings or a list of floats.""" if not self.is_test: # Check timing validity # expect to see a monotonic increase in the relative times - is_monotonic = np.all(np.diff(relative_times) >= 0) + is_monotonic = np.all(np.diff(self.message_data['timing_series']) >= 0) if not is_monotonic: raise ValueError(f'{self.__class__.__name__} timing_series must be in increasing order. ' f'They represent the time after initial neutrino time') diff --git a/snews_pt/test/test_timing_tier.py b/snews_pt/test/test_timing_tier.py index 89b435b..e4c7ba6 100644 --- a/snews_pt/test/test_timing_tier.py +++ b/snews_pt/test/test_timing_tier.py @@ -5,7 +5,9 @@ def test_timing_expected(): """Test with example of expected message type.""" # Create timing tier message. tims = SNEWSMessageBuilder(detector_name='XENONnT', - timing_series=['2012-06-09T15:31:08.109876', '2012-06-09T15:33:07.891011'], + timing_series=['2012-06-09T15:31:08.109876', '2012-06-09T15:33:07.891011', + '2012-06-09T15:33:07.9910110', '2012-06-09T15:34:07.891011000', + '2012-06-09T15:35:17.0'], machine_time='2012-06-09T15:30:00.009876', firedrill_mode=False, is_test=True) @@ -17,8 +19,38 @@ def test_timing_expected(): 'machine_time': '2012-06-09T15:30:00.009876000', 'p_val': None, 'is_test': True, - 'timing_series': ['2012-06-09T15:31:08.109876000', - '2012-06-09T15:33:07.891011000']} + 'timing_series': [0, 119781135000, 119881135000, + 179781135000, 248890124000]} + assert tims.messages[0].meta == { 'firedrill_mode': False} + + assert tims.messages[0].is_valid() is True, "There are invalid messages" + + # Try to send message to SNEWS 2.0 server. + try: + tims.send_messages() + except Exception as exc: + print('SNEWSMessageBuilder.send_messages() test failed!\n') + assert False, f"Exception raised:\n {exc}" + +def test_timing_expected_with_floats(): + """Test with example of expected message type.""" + # Create timing tier message. + tims = SNEWSMessageBuilder(detector_name='XENONnT', + timing_series=[0, 119781135000, 119881135000, + 179781135000, 248890124000], + machine_time='2012-06-09T15:30:00.009876', + firedrill_mode=False, is_test=True) + + # # Check that message has expected structure. + assert tims.selected_tiers == ['SNEWSTimingTierMessage'] + assert tims.messages[0].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876000', + 'schema_version': '1.3.1', + 'detector_name': 'XENONnT', + 'machine_time': '2012-06-09T15:30:00.009876000', + 'p_val': None, + 'is_test': True, + 'timing_series': [0, 119781135000, 119881135000, + 179781135000, 248890124000]} assert tims.messages[0].meta == { 'firedrill_mode': False} assert tims.messages[0].is_valid() is True, "There are invalid messages" From 86e0f563886a610a257078733c7048fbde6bfc11 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Tue, 7 May 2024 14:19:00 +0200 Subject: [PATCH 08/10] placeholder for prod topic --- snews_pt/auxiliary/test-config.env | 1 + 1 file changed, 1 insertion(+) diff --git a/snews_pt/auxiliary/test-config.env b/snews_pt/auxiliary/test-config.env index 6835628..3792ce3 100644 --- a/snews_pt/auxiliary/test-config.env +++ b/snews_pt/auxiliary/test-config.env @@ -14,6 +14,7 @@ HOP_BROKER="kafka.scimma.org" OBSERVATION_TOPIC="kafka://${HOP_BROKER}/snews.experiments-test" ALERT_TOPIC="kafka://${HOP_BROKER}/snews.alert-test" +PRODUCTION_TOPIC="" FIREDRILL_OBSERVATION_TOPIC="kafka://${HOP_BROKER}/snews.experiments-firedrill" FIREDRILL_ALERT_TOPIC="kafka://${HOP_BROKER}/snews.alert-firedrill" From 713549b37b5f45931c631234a8f77e288dfdeb94 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Tue, 7 May 2024 15:21:11 +0200 Subject: [PATCH 09/10] add neutrino times for time tier --- snews_pt/messages.py | 24 ++++++++++++++---------- snews_pt/test/test_timing_tier.py | 24 +++++++++++++++--------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index d3a98ee..058ce0e 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -307,30 +307,34 @@ def is_valid(self): class SNEWSTimingTierMessage(SNEWSMessage): - """Message for SNEWS 2.0 timing tier.""" + """ Message for SNEWS 2.0 timing tier. + `timing_series` can either be list of string or nanosecond-precision integers + representing the time after the initial neutrino time. + """ - reqfields = [ 'timing_series' ] + reqfields = [ 'timing_series', 'neutrino_time' ] fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val', 'is_test' ] - def __init__(self, p_val=None, timing_series=None, **kwargs): + def __init__(self, p_val=None, timing_series=None, neutrino_time=None, **kwargs): + initial_nu_time_str = clean_time_input(neutrino_time) + initial_nu_time_object = np.datetime64(initial_nu_time_str).astype('datetime64[ns]') # first convert the timing series into relative times - times = self.convert_times(timing_series) + times = self._convert_times(timing_series, initial_neutrino_time=initial_nu_time_object) super().__init__(self.fields, p_val=p_val, timing_series=times, + neutrino_time=initial_nu_time_str, **kwargs) - def convert_times(self, timing_series): + def _convert_times(self, timing_series, initial_neutrino_time): if all([isinstance(t, str) for t in timing_series]): # convert to numpy datetime objects times_obj = np.array([np.datetime64(t) for t in timing_series]).astype('datetime64[ns]') times_obj = np.sort(times_obj) - relative_times = (times_obj - times_obj[0]).astype('timedelta64[ns]') # make sure they are always ns precision - relative_times = [int(t//np.timedelta64(1, 'ns')) for t in relative_times] + initial_neutrino_time = times_obj[0] if initial_neutrino_time is None else initial_neutrino_time + # make sure they are always ns precision + relative_times = (times_obj - initial_neutrino_time).astype('timedelta64[ns]').astype(int).tolist() elif all([isinstance(t, (int, float)) for t in timing_series]): - # if they are relative, expect an initial neutrino time - # if "neutrino_time" not in self.meta: - # raise ValueError(f'{self.__class__.__name__} neutrino_time must be provided if timing_series are relative.') # then we assume they are relative times from the first neutrino time with ns precision relative_times = timing_series if isinstance(timing_series, list) else list(timing_series) else: diff --git a/snews_pt/test/test_timing_tier.py b/snews_pt/test/test_timing_tier.py index e4c7ba6..ccd7310 100644 --- a/snews_pt/test/test_timing_tier.py +++ b/snews_pt/test/test_timing_tier.py @@ -2,9 +2,12 @@ from snews_pt.messages import SNEWSMessageBuilder def test_timing_expected(): - """Test with example of expected message type.""" + """Test with example of expected message type. + This test passes the neutrino times in strings, and along with an additional initial neutrino time. + """ # Create timing tier message. tims = SNEWSMessageBuilder(detector_name='XENONnT', + neutrino_time='2012-06-09T15:31:08.109876', timing_series=['2012-06-09T15:31:08.109876', '2012-06-09T15:33:07.891011', '2012-06-09T15:33:07.9910110', '2012-06-09T15:34:07.891011000', '2012-06-09T15:35:17.0'], @@ -12,18 +15,19 @@ def test_timing_expected(): firedrill_mode=False, is_test=True) # # Check that message has expected structure. - assert tims.selected_tiers == ['SNEWSTimingTierMessage'] - assert tims.messages[0].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876000', + assert tims.selected_tiers == ['SNEWSCoincidenceTierMessage', 'SNEWSTimingTierMessage'] + assert tims.messages[1].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876000', 'schema_version': '1.3.1', 'detector_name': 'XENONnT', 'machine_time': '2012-06-09T15:30:00.009876000', + 'neutrino_time': '2012-06-09T15:31:08.109876000', 'p_val': None, 'is_test': True, 'timing_series': [0, 119781135000, 119881135000, 179781135000, 248890124000]} - assert tims.messages[0].meta == { 'firedrill_mode': False} + assert tims.messages[1].meta == { 'firedrill_mode': False} - assert tims.messages[0].is_valid() is True, "There are invalid messages" + assert tims.messages[1].is_valid() is True, "There are invalid messages" # Try to send message to SNEWS 2.0 server. try: @@ -36,24 +40,26 @@ def test_timing_expected_with_floats(): """Test with example of expected message type.""" # Create timing tier message. tims = SNEWSMessageBuilder(detector_name='XENONnT', + neutrino_time='2012-06-09T15:31:08.109876', timing_series=[0, 119781135000, 119881135000, 179781135000, 248890124000], machine_time='2012-06-09T15:30:00.009876', firedrill_mode=False, is_test=True) # # Check that message has expected structure. - assert tims.selected_tiers == ['SNEWSTimingTierMessage'] - assert tims.messages[0].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876000', + assert tims.selected_tiers == ['SNEWSCoincidenceTierMessage', 'SNEWSTimingTierMessage'] + assert tims.messages[1].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876000', 'schema_version': '1.3.1', 'detector_name': 'XENONnT', 'machine_time': '2012-06-09T15:30:00.009876000', + 'neutrino_time': '2012-06-09T15:31:08.109876000', 'p_val': None, 'is_test': True, 'timing_series': [0, 119781135000, 119881135000, 179781135000, 248890124000]} - assert tims.messages[0].meta == { 'firedrill_mode': False} + assert tims.messages[1].meta == { 'firedrill_mode': False} - assert tims.messages[0].is_valid() is True, "There are invalid messages" + assert tims.messages[1].is_valid() is True, "There are invalid messages" # Try to send message to SNEWS 2.0 server. try: From 4db8f999f313cb52b00cbdc4c39c2dabb5e05598 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Tue, 7 May 2024 15:43:21 +0200 Subject: [PATCH 10/10] clean a line --- snews_pt/messages.py | 1 - 1 file changed, 1 deletion(-) diff --git a/snews_pt/messages.py b/snews_pt/messages.py index 058ce0e..7eaa8d8 100644 --- a/snews_pt/messages.py +++ b/snews_pt/messages.py @@ -331,7 +331,6 @@ def _convert_times(self, timing_series, initial_neutrino_time): # convert to numpy datetime objects times_obj = np.array([np.datetime64(t) for t in timing_series]).astype('datetime64[ns]') times_obj = np.sort(times_obj) - initial_neutrino_time = times_obj[0] if initial_neutrino_time is None else initial_neutrino_time # make sure they are always ns precision relative_times = (times_obj - initial_neutrino_time).astype('timedelta64[ns]').astype(int).tolist() elif all([isinstance(t, (int, float)) for t in timing_series]):