SQL Database Compatibility

docs/user/command_line_interface.md

@@ -76,15 +76,14 @@ instances where this data has been read but could not be send to server right aw
 ```
 In which case it displays the following
 ```bash
-         >The Message Schema for TimeTier
-         _id                 :(SNEWS SETS)
-         schema_version      :(SNEWS SETS)
-         detector_name       :(FETCHED FROM ENV XENONnT)
-         machine_time        :(User Input)
-         neutrino_time       :(User Input)
-         p_val               :(User Input)
-         timing_series       :(User Input*)
-         **kwargs            :(APPENDED AS 'META') 
+         > Message schema for SNEWSTimingTierMessage                                                                               
+         _id                  : (SET AUTOMATICALLY)
+         schema_version       : (SET AUTOMATICALLY)
+         detector_name        : (SET AUTOMATICALLY)
+         timing_series        : (REQUIRED USER INPUT)
+         machine_time         : (USER INPUT)
+         p_val                : (USER INPUT)
+         is_test              : (USER INPUT)
 ```
 or you can simply call `snews_pt message-schema all`  to display all the message schemes. <br>
 

snews_pt/__main__.py

@@ -106,7 +106,8 @@ def message_schema(ctx, requested_tier):
         displays everything
     """
     from .messages import SNEWSHeartbeatMessage, SNEWSTimingTierMessage, SNEWSSignificanceTierMessage, \
-        SNEWSCoincidenceTierMessage, SNEWSRetractionMessage
+        SNEWSCoincidenceTierMessage, SNEWSRetractionMessage, SNEWSMessage
+
     tier_data_pairs = {'CoincidenceTier': SNEWSCoincidenceTierMessage,
                        'SigTier': SNEWSSignificanceTierMessage,
                        'TimeTier': SNEWSTimingTierMessage,
@@ -123,19 +124,22 @@ def message_schema(ctx, requested_tier):
             tier = list(tier_data_pairs.keys())
         else:
             # check for aliases e.g. coinc = coincidence = CoinCideNceTier
-            tier = snews_pt_utils._check_aliases(requested_tier[0])
+            tier = list(snews_pt_utils._check_aliases(requested_tier[0]))
 
+    basefields = SNEWSMessage.basefields
     for t in tier:
         TierMessage = tier_data_pairs[t]
-        click.secho(f'Message schema for {TierMessage.__name__}', bg='white', fg='blue')
-        for f in TierMessage.fields:
-            if f in TierMessage.basefields:
+        fields = TierMessage.fields
+        reqfields = TierMessage.reqfields
+        click.secho(f'Message schema for {t}', bg='white', fg='blue')
+        for f in fields:
+            if f in basefields:
                 click.secho(f'{f:<20s} : (SET AUTOMATICALLY)', fg='bright_red')
-            elif f in TierMessage.reqfields:
+            elif f in reqfields:
                 click.secho(f'{f:<20s} : (REQUIRED USER INPUT)', fg='bright_blue')
             else:
                 click.secho(f'{f:<20s} : (USER INPUT)', fg='bright_cyan')
-
+        click.secho(f'{"**kwargs":<20s} : (GROUPED AS META)', fg='bright_green')
 
 @main.command()
 @click.option('--firedrill/--no-firedrill', default=True, show_default='True', help='Whether to use firedrill brokers or default ones')

snews_pt/_version.py

@@ -1 +1 @@
-version = '1.3.0'
+version = '1.3.1'

snews_pt/messages.py

@@ -129,6 +129,7 @@ def __init__(self, fields, detector_name='TEST', **kwargs):
             machine_time = raw_mt
             )
 
+        self.is_test = kwargs.get('is_test', False)
         for kw in kwargs:
             if kw in fields:
                 # Append all kwargs matching subclass fields in message_data.
@@ -137,7 +138,6 @@ def __init__(self, fields, detector_name='TEST', **kwargs):
                 # Append all non-matching kwargs to meta.
                 self.meta[kw] = kwargs[kw]
 
-        self.is_test = kwargs.get('is_test', False)
         # Check that required fields are present and valid.
         self.has_required_fields()
 
@@ -150,6 +150,7 @@ def print_schema(self):
                 click.secho(f'{f:<20s} : (REQUIRED USER INPUT)', fg='bright_blue')
             else:
                 click.secho(f'{f:<20s} : (USER INPUT)', fg='bright_cyan')
+        click.secho(f'{"**kwargs":<20s} : (GROUPED AS META)', fg='bright_green')
 
     def get_detector_name(self, detector_name):
         """Get formatted detector name.
@@ -166,6 +167,8 @@ def get_detector_name(self, detector_name):
         """
         if detector_name == 'TEST' or detector_name is None:
             detector_name = snews_pt_utils.get_name()
+        else:
+            detector_name = snews_pt_utils.set_name(detector_name, _return=True)
         return detector_name
 
     def clean_time_input(self, time):
@@ -242,7 +245,7 @@ class SNEWSCoincidenceTierMessage(SNEWSMessage):
     """Message for SNEWS 2.0 coincidence tier."""
 
     reqfields = [ 'neutrino_time' ]
-    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val' ]
+    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val', 'is_test' ]
 
     def __init__(self, neutrino_time=None, p_val=None, **kwargs):
         super().__init__(self.fields,
@@ -274,13 +277,9 @@ class SNEWSSignificanceTierMessage(SNEWSMessage):
     """Message for SNEWS 2.0 significance tier."""
 
     reqfields = [ 'p_values', 't_bin_width' ]
-    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time' ]
+    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'is_test' ]
 
     def __init__(self, p_values=None, t_bin_width=None, **kwargs):
-        # Type check for proper types.
-        if np.isscalar(p_values):
-            raise RuntimeError(f'{self.__class__.__name__} p_values must be a list.')
-
         super().__init__(self.fields,
                          p_values=p_values,
                          t_bin_width=t_bin_width,
@@ -289,6 +288,9 @@ def __init__(self, p_values=None, t_bin_width=None, **kwargs):
     def is_valid(self):
         """Check that parameter values are valid for this tier."""
         if not self.is_test:
+            # Type check for proper types.
+            if np.isscalar(self.message_data['p_values']):
+                raise RuntimeError(f'{self.__class__.__name__} p_values must be a list.')
             for pv in self.message_data['p_values']:
                 if isinstance(pv, str):
                     pv = float(pv)
@@ -299,14 +301,16 @@ def is_valid(self):
                     raise ValueError(f'{self.__class__.__name__} t_bin_width must be a float.')
             elif not isinstance(self.message_data['t_bin_width'], float):
                 raise ValueError(f'{self.__class__.__name__} t_bin_width must be a float.')
+
+
         return True
 
 
 class SNEWSTimingTierMessage(SNEWSMessage):
     """Message for SNEWS 2.0 timing tier."""
 
     reqfields = [ 'timing_series' ]
-    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val' ]
+    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'p_val', 'is_test' ]
 
     def __init__(self, p_val=None, timing_series=None, **kwargs):
         super().__init__(self.fields,
@@ -326,14 +330,21 @@ def is_valid(self):
                 duration = (timeobj - datetime.utcnow()).total_seconds()
                 if (duration <= -172800.0) or (duration > 0.0):
                     raise ValueError(f'{self.__class__.__name__} neutrino_time must be within 48 hours of now.')
+
+                # p_val must be a float between 0 and 1
+                pv = self.message_data['p_val']
+                if isinstance(pv, str):
+                    pv = float(pv)
+                if not (0.0 <= pv <= 1.0):
+                    raise ValueError(f'{self.__class__.__name__} p_value of the detection must be between 0 and 1.')
         return True
 
 
 class SNEWSRetractionMessage(SNEWSMessage):
     """Message for SNEWS 2.0 retractions."""
 
     reqfields = [ 'retract_latest' ]
-    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'retraction_reason' ]
+    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'retraction_reason', 'is_test' ]
 
     def __init__(self, retract_latest=None, retraction_reason=None, **kwargs):
         super().__init__(self.fields,
@@ -352,7 +363,7 @@ class SNEWSHeartbeatMessage(SNEWSMessage):
     """Message for SNEWS 2.0 heartbeats."""
 
     reqfields = [ 'detector_status' ]
-    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time' ]
+    fields = SNEWSMessage.basefields + reqfields + [ 'machine_time', 'is_test' ]
 
     def __init__(self, machine_time=None, detector_status=None, **kwargs):
         super().__init__(self.fields,
@@ -481,12 +492,16 @@ def send_messages(self, firedrill_mode=True, env_file=None, verbose=True, auth=T
         messages_to_send = []
 
         # append meta fields if the meta field is not in the other messages
+        meta_dict = {}
         for m in self.messages:
             mes = m.message_data
             met = m.meta
             for k, v in met.items():
+                # store meta fields to append later
+                # if the meta field is not in the other messages
                 if k not in fields_set:
-                    mes[k] = v
+                    meta_dict[k] = v
+            mes["meta"] = meta_dict
             messages_to_send.append(mes)
 
         with Publisher(env_path=env_file,

snews_pt/test/test_coincidence_tier.py

@@ -14,10 +14,13 @@ def test_coincidence_expected():
     assert coin.messages[0].message_data == {'_id': 'KamLAND_CoincidenceTier_2012-06-09T15:30:00.000501',
                                              'detector_name': 'KamLAND',
                                              'machine_time': '2012-06-09T15:30:00.000501',
-                                             'schema_version': '1.3.0',
+                                             'schema_version': '1.3.1',
                                              'neutrino_time': '2012-06-09T15:31:08.891011',
+                                             'is_test': True,
                                              'p_val': None}
-    assert coin.messages[0].meta == {'is_test': True, 'firedrill_mode': False}
+
+    # firedrill_mode is an argument of coinc.send_messages(), so it becomes meta here
+    assert coin.messages[0].meta == {'firedrill_mode': False}
 
     # check if valid snews format
     assert coin.messages[0].is_valid() is True, "Message is not valid"

snews_pt/test/test_heartbeat.py

@@ -12,11 +12,14 @@ def test_heartbeat_expected():
     assert len(hb.messages) == 1, f"Expected 1 Heartbeat Message got {len(hb.messages)}!"
 
     assert hb.messages[0].message_data == {'_id': 'XENONnT_Heartbeat_2012-06-09T15:30:00.000501',
-                                           'schema_version': '1.3.0',
+                                           'schema_version': '1.3.1',
                                            'detector_name': 'XENONnT',
                                            'machine_time': '2012-06-09T15:30:00.000501',
-                                           'detector_status': 'ON'}
-    assert hb.messages[0].meta == {'is_test': True, 'firedrill_mode': False}
+                                           'detector_status': 'ON',
+                                           'is_test': True}
+
+    # firedrill_mode is an argument of hb.send_messages(), so it becomes meta here
+    assert hb.messages[0].meta == {'firedrill_mode': False}
 
     # # check if valid snews format
     assert hb.messages[0].is_valid() is True, "Message is not valid"

snews_pt/test/test_retraction.py

@@ -23,12 +23,15 @@ def test_retraction():
                                              is_test=True, firedrill_mode=False)
     assert retraction_message.selected_tiers == ["SNEWSRetractionMessage"]
     assert retraction_message.messages[0].message_data == {'_id': 'KamLAND_Retraction_2012-06-09T15:30:00.000501',
-                                                           'schema_version': '1.3.0',
+                                                           'schema_version': '1.3.1',
                                                            'detector_name': 'KamLAND',
                                                            'machine_time': '2012-06-09T15:30:00.000501',
                                                            'retract_latest': 1,
-                                                           'retraction_reason': None}, "created message data is wrong"
-    assert retraction_message.messages[0].meta == {'is_test': True, 'firedrill_mode': False}, "created meta is wrong"
+                                                           'retraction_reason': None,
+                                                           'is_test': True}, "created message data is wrong"
+
+    # firedrill_mode is an argument of retraction_message.send_messages(), so it becomes meta here
+    assert retraction_message.messages[0].meta == {'firedrill_mode': False}, "created meta is wrong"
     assert retraction_message.messages[0].is_valid() is True, "Invalid retraction message created"
 
     try:

snews_pt/test/test_significance_tier.py

@@ -17,31 +17,31 @@ def test_significance_expected():
     # Check that message has expected structure.
     assert sign.selected_tiers == ['SNEWSCoincidenceTierMessage', 'SNEWSSignificanceTierMessage']
     assert sign.messages[0].message_data == {'_id': 'DS-20K_CoincidenceTier_2012-06-09T15:30:00.000501',
-                                             'schema_version': '1.3.0',
+                                             'schema_version': '1.3.1',
                                              'detector_name': 'DS-20K',
                                              'machine_time': '2012-06-09T15:30:00.000501',
                                              'neutrino_time': '2012-06-09T15:31:08.109876',
-                                             'p_val': None}
+                                             'p_val': None,
+                                             'is_test': True}
 
     assert sign.messages[0].meta == {'p_values': [0.4, 0.5],
                                      't_bin_width': 0.8,
-                                     'is_test': True,
                                      'neutrino_times': ['2012-06-09T15:31:08.109876',
                                                         '2012-06-09T15:33:07.891098'],
                                      'firedrill_mode': False}
 
     assert sign.messages[1].message_data == {'_id': 'DS-20K_SignificanceTier_2012-06-09T15:30:00.000501',
-                                             'schema_version': '1.3.0',
+                                             'schema_version': '1.3.1',
                                              'detector_name': 'DS-20K',
                                              'machine_time': '2012-06-09T15:30:00.000501',
                                              'p_values': [0.4, 0.5],
-                                             't_bin_width': 0.8}
+                                             't_bin_width': 0.8,
+                                             'is_test': True}
 
     assert sign.messages[1].meta == {'neutrino_time': '2012-06-09T15:31:08.109876',
-                                     'is_test': True,
                                      'neutrino_times': ['2012-06-09T15:31:08.109876',
                                                         '2012-06-09T15:33:07.891098'],
-                                     'firedrill_mode': False}
+                                     'firedrill_mode': False,}
 
     assert sign.messages[0].is_valid() is True, "invalid coincidence tier message in 'test_significance_tier'"
     assert sign.messages[1].is_valid() is True, "invalid significance tier message in 'test_significance_tier'"

snews_pt/test/test_timing_tier.py

@@ -12,13 +12,14 @@ def test_timing_expected():
     # # Check that message has expected structure.
     assert tims.selected_tiers == ['SNEWSTimingTierMessage']
     assert tims.messages[0].message_data == {'_id': 'XENONnT_TimingTier_2012-06-09T15:30:00.009876',
-                                             'schema_version': '1.3.0',
+                                             'schema_version': '1.3.1',
                                              'detector_name': 'XENONnT',
                                              'machine_time': '2012-06-09T15:30:00.009876',
                                              'p_val': None,
+                                             'is_test': True,
                                              'timing_series': ['2012-06-09T15:31:08.109876',
                                                                '2012-06-09T15:33:07.891011']}
-    assert tims.messages[0].meta == {'is_test': True, 'firedrill_mode': False}
+    assert tims.messages[0].meta == { 'firedrill_mode': False}
 
     assert tims.messages[0].is_valid() is True, "There are invalid messages"