From 9621c862fe550839b573f57a305b16d70b48c674 Mon Sep 17 00:00:00 2001 From: Segev BenZvi Date: Thu, 15 Aug 2024 10:14:08 -0500 Subject: [PATCH 01/16] Add test of server connection. --- snews_cs/test/test_01_connection_to_server.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 snews_cs/test/test_01_connection_to_server.py diff --git a/snews_cs/test/test_01_connection_to_server.py b/snews_cs/test/test_01_connection_to_server.py new file mode 100644 index 0000000..b9ea696 --- /dev/null +++ b/snews_cs/test/test_01_connection_to_server.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +"""Initialization unit tests for the snews_cs module. +""" +import unittest + +import snews_cs +import snews_pt.remote_commands as sptrc + +import io +import contextlib + +class TestServer(unittest.TestCase): + def test_connection(self): + f = io.StringIO() + with contextlib.redirect_stdout(f): + #- Connect to server + sptrc.test_connection(detector_name='XENONnT', firedrill=False, start_at='LATEST', patience=8) + + #- Check the output message; it should say "You (XENONnT) have a connection" + confirm_msg = 'You (XENONnT) have a connection to the server' + self.assertTrue(confirm_msg in f.getvalue()) From 17de2ffd8b9adb49b7b9dfb5c9030a93ae7fa66d Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 10:25:43 -0500 Subject: [PATCH 02/16] add hop auth --- .github/workflows/tests.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e03175..a33cafa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,7 +52,28 @@ jobs: - name: Install SNEWS_Coincidence_System run: | python -m poetry install - + + - shell: bash + env: + USERNAME: ${{ secrets.hop_user_name }} + PASSWORD: ${{ secrets.hop_user_password }} + run: | + sudo apt-get install -y expect + which expect + /usr/bin/expect << HOP + spawn hop auth add + expect "Username:" + send "$USERNAME\n" + expect "Password:" + send "$PASSWORD\n" + expect "Hostname (may be empty):" + send "kafka.scimma.org\n" + expect "Token endpoint (empty if not applicable):" + send "\n" + expect eof + HOP + hop auth locate + # Run the unit tests - name: Test with pytest run: | From e6550647382a4cd5a06ea06e755ffeeed08f3617 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 10:26:30 -0500 Subject: [PATCH 03/16] identation --- snews_cs/test/test_alert.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 snews_cs/test/test_alert.py diff --git a/snews_cs/test/test_alert.py b/snews_cs/test/test_alert.py new file mode 100644 index 0000000..67ae36e --- /dev/null +++ b/snews_cs/test/test_alert.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +"""Initialization unit tests for the snews_cs module. +""" +import unittest +import snews_cs +from snews_pt.messages import SNEWSMessageBuilder + +def test_alert(unittest.TestCase): + """Test alert message builder.""" + message1 = SNEWSMessageBuilder(detector_name="XENONnT", + machine_time="2022-01-01T00:00:00.000", + detector_status="ON",) + + + From 3d69d02e864e76769d648db291e579c3e8f683d6 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 10:26:47 -0500 Subject: [PATCH 04/16] indentation --- .github/workflows/tests.yml | 40 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a33cafa..e21657e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,26 +53,26 @@ jobs: run: | python -m poetry install - - shell: bash - env: - USERNAME: ${{ secrets.hop_user_name }} - PASSWORD: ${{ secrets.hop_user_password }} - run: | - sudo apt-get install -y expect - which expect - /usr/bin/expect << HOP - spawn hop auth add - expect "Username:" - send "$USERNAME\n" - expect "Password:" - send "$PASSWORD\n" - expect "Hostname (may be empty):" - send "kafka.scimma.org\n" - expect "Token endpoint (empty if not applicable):" - send "\n" - expect eof - HOP - hop auth locate + - shell: bash + env: + USERNAME: ${{ secrets.hop_user_name }} + PASSWORD: ${{ secrets.hop_user_password }} + run: | + sudo apt-get install -y expect + which expect + /usr/bin/expect << HOP + spawn hop auth add + expect "Username:" + send "$USERNAME\n" + expect "Password:" + send "$PASSWORD\n" + expect "Hostname (may be empty):" + send "kafka.scimma.org\n" + expect "Token endpoint (empty if not applicable):" + send "\n" + expect eof + HOP + hop auth locate # Run the unit tests - name: Test with pytest From 9587fc48750b4226a8b736eb0b9c43ce8360b3a0 Mon Sep 17 00:00:00 2001 From: Segev BenZvi Date: Thu, 15 Aug 2024 10:31:56 -0500 Subject: [PATCH 05/16] Remove test alert. --- snews_cs/test/test_alert.py | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 snews_cs/test/test_alert.py diff --git a/snews_cs/test/test_alert.py b/snews_cs/test/test_alert.py deleted file mode 100644 index 67ae36e..0000000 --- a/snews_cs/test/test_alert.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -"""Initialization unit tests for the snews_cs module. -""" -import unittest -import snews_cs -from snews_pt.messages import SNEWSMessageBuilder - -def test_alert(unittest.TestCase): - """Test alert message builder.""" - message1 = SNEWSMessageBuilder(detector_name="XENONnT", - machine_time="2022-01-01T00:00:00.000", - detector_status="ON",) - - - From 8b3ed5db1975a393e6a15b9fc2155fb537b6e48d Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 10:34:22 -0500 Subject: [PATCH 06/16] install hop step --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e21657e..f1761a9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,15 @@ jobs: run: | python -m poetry install + - name: Install hop-client + run: | + pip install setuptools wheel + wget https://files.pythonhosted.org/packages/64/d1/108cea042128c7ea7790e15e12e3e5ed595bfcf4b051c34fe1064924beba/hop-client-0.9.0.tar.gz + tar -xzf hop-client-0.9.0.tar.gz + cd hop-client-0.9.0 + python setup.py install + cd /home/runner/work/SNEWS_Publishing_Tools/SNEWS_Publishing_Tools + - shell: bash env: USERNAME: ${{ secrets.hop_user_name }} From f25ed23662e0867eab02ec5227d81efa860b5f49 Mon Sep 17 00:00:00 2001 From: Segev BenZvi Date: Thu, 15 Aug 2024 10:37:15 -0500 Subject: [PATCH 07/16] Update folder name. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f1761a9..8ec719b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: tar -xzf hop-client-0.9.0.tar.gz cd hop-client-0.9.0 python setup.py install - cd /home/runner/work/SNEWS_Publishing_Tools/SNEWS_Publishing_Tools + cd /home/runner/work/SNEWS_Coincidence_System/SNEWS_Coincidence_System - shell: bash env: From e1a7ae94a093a7596ec8e89ab649b6a05bda0f90 Mon Sep 17 00:00:00 2001 From: Segev BenZvi Date: Thu, 15 Aug 2024 10:42:15 -0500 Subject: [PATCH 08/16] Only run snews_cs tests. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ec719b..732fc7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,4 +86,4 @@ jobs: # Run the unit tests - name: Test with pytest run: | - python -m poetry run pytest + python -m poetry run pytest snews_cs From ede28a4122e3846e477127fb065532854cbab0a2 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 14:56:50 -0500 Subject: [PATCH 09/16] implement a test cache and topic --- snews_cs/snews_coinc.py | 124 ++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 36 deletions(-) diff --git a/snews_cs/snews_coinc.py b/snews_cs/snews_coinc.py index d5c55e7..34e3d95 100644 --- a/snews_cs/snews_coinc.py +++ b/snews_cs/snews_coinc.py @@ -409,10 +409,13 @@ def __init__(self, env_path=None, drop_db=False, firedrill_mode=True, hb_path=No self.exit_on_error = False # True self.initial_set = False self.alert = AlertPublisher(env_path=env_path, firedrill_mode=firedrill_mode) + self.test_alert = AlertPublisher(env_path=env_path, is_test=True) # overwrites with connection test topic if firedrill_mode: self.observation_topic = os.getenv("FIREDRILL_OBSERVATION_TOPIC") else: self.observation_topic = os.getenv("OBSERVATION_TOPIC") + # for testing, the alerts will be sent to this topic + self.test_topic = os.getenv("CONNECTION_TEST_TOPIC") self.alert_schema = CoincidenceTierAlert(env_path) # handle heartbeat self.store_heartbeat = bool(os.getenv("STORE_HEARTBEAT", "True")) @@ -420,7 +423,10 @@ def __init__(self, env_path=None, drop_db=False, firedrill_mode=True, hb_path=No self.stash_time = 86400 self.coinc_data = CacheManager() + self.test_coinc_data = CacheManager() # a separate cache for testing self.message_count = {} + self.test_message_count = {} + ## don't use a storage for the test cache def clear_cache(self): """ When a reset cache is passed, recreate the @@ -432,7 +438,7 @@ def clear_cache(self): self.coinc_data = CacheManager() # ---------------------------------------------------------------------------------------------------------------- - def display_table(self): + def display_table(self, which_cache_to_use): """ Display each sub list individually using a markdown table. @@ -440,21 +446,32 @@ def display_table(self): click.secho( f'Here is the current coincident table\n', fg='magenta', bold=True, ) - for sub_list in self.coinc_data.cache['sub_group'].unique(): - sub_df = self.coinc_data.cache.query(f'sub_group=={sub_list}') + if which_cache_to_use == 'main': + cache_data = self.coinc_data + else: + cache_data = self.test_coinc_data + for sub_list in cache_data.cache['sub_group'].unique(): + sub_df = cache_data.cache.query(f'sub_group=={sub_list}') sub_df = sub_df.drop(columns=['meta', 'machine_time', 'schema_version', 'neutrino_time_as_datetime']) sub_df = sub_df.sort_values(by=['neutrino_time']) # snews_bot.send_table(sub_df) # no need to print the table on the server. Logs have the full content print(sub_df.to_markdown()) print('=' * 168) - def send_alert(self, sub_group_tag, alert_type): - sub_df = self.coinc_data.cache.query('sub_group==@sub_group_tag') + def send_alert(self, sub_group_tag, alert_type, which_cache_to_use="main"): + if which_cache_to_use == 'main': + sub_df = self.coinc_data.cache.query('sub_group==@sub_group_tag') + false_alarm_prob = cache_false_alarm_rate(cache_sub_list=sub_df, hb_cache=self.heartbeat.cache_df) + alert_publisher = self.alert + else: + sub_df = self.test_coinc_data.cache.query('sub_group==@sub_group_tag') + false_alarm_prob = "N/A" + alert_publisher = self.test_alert + p_vals = sub_df['p_val'].to_list() p_vals_avg = np.round(sub_df['p_val'].mean(), decimals=5) nu_times = sub_df['neutrino_time'].to_list() detector_names = sub_df['detector_name'].to_list() - false_alarm_prob = cache_false_alarm_rate(cache_sub_list=sub_df, hb_cache=self.heartbeat.cache_df) alert_data = dict(p_vals=p_vals, p_val_avg=p_vals_avg, sub_list_num=int(sub_group_tag), @@ -464,19 +481,22 @@ def send_alert(self, sub_group_tag, alert_type): server_tag=self.server_tag, alert_type=alert_type) - with self.alert as pub: + + with alert_publisher as pub: alert = self.alert_schema.get_cs_alert_schema(data=alert_data) pub.send(alert) - if self.send_email: - send_email(alert) - if self.send_slack: - snews_bot.send_table(alert_data, - alert, - is_test=True, - topic=self.observation_topic) + # only check to see if email or slack should be sent if the alert is not a test alert + if which_cache_to_use == 'main': + if self.send_email: + send_email(alert) + if self.send_slack: + snews_bot.send_table(alert_data, + alert, + is_test=True, + topic=self.observation_topic) # ------------------------------------------------------------------------------------------------------------------ - def alert_decider(self): + def alert_decider(self, which_cache_to_use="main"): """ This method will publish an alert every time a new detector submits an observation message @@ -487,7 +507,16 @@ def alert_decider(self): # loop through the sub group tag and state # print(f'TEST {self.coinc_data.sub_group_state}') - for sub_group_tag, state in self.coinc_data.sub_group_state.items(): + # decide which cache to use + if which_cache_to_use == 'main': + cache_data = self.coinc_data + _message_count = self.message_count + # send_alert arg + else: + cache_data = self.test_coinc_data + _message_count = self.test_message_count + + for sub_group_tag, state in cache_data.sub_group_state.items(): print('CHECKING FOR ALERTS IN SUB GROUP: ', sub_group_tag) # if state is none skip the sub group if state is None: @@ -505,7 +534,7 @@ def alert_decider(self): self.send_alert(sub_group_tag=sub_group_tag, alert_type=state) continue # publish a retraction alert for the sub group is its state is RETRACTION - elif state == 'RETRACTION' and len(self.coinc_data.cache.query('sub_group==@sub_group_tag')) < self.message_count[sub_group_tag]: + elif state == 'RETRACTION' and len(cache_data.cache.query('sub_group==@sub_group_tag')) < _message_count[sub_group_tag]: # yet another pretty terminal output click.secho(f'SUB GROUP {sub_group_tag}:{"RETRACTION HAS BEEN MADE".upper():^100}', bg='bright_green', fg='red') @@ -522,20 +551,20 @@ def alert_decider(self): fg='red') click.secho(f'{"=" * 100}', fg='bright_red') continue - elif state == 'UPDATE' and len(self.coinc_data.cache.query('sub_group==@sub_group_tag')) == self.message_count[sub_group_tag]: + elif state == 'UPDATE' and len(cache_data.cache.query('sub_group==@sub_group_tag')) == _message_count[sub_group_tag]: # yet another pretty terminal output click.secho(f'SUB GROUP {sub_group_tag}:{"A MESSAGE HAS BEEN UPDATED".upper():^100}', bg='bright_green', fg='red') log.debug('\t> An UPDATE message is received') # only publish an alert if the sub group has more than 1 message - if len(self.coinc_data.cache.query('sub_group==@sub_group_tag')) > 1: + if len(cache_data.cache.query('sub_group==@sub_group_tag')) > 1: click.secho(f'{"Publishing an updated Alert!!!".upper():^100}', bg='bright_green', fg='red') click.secho(f'{"=" * 100}', fg='bright_red') # publish update alert self.send_alert(sub_group_tag=sub_group_tag, alert_type=state) log.debug('\t> An alert is updated!') continue - elif state == 'COINC_MSG' and len(self.coinc_data.cache.query('sub_group==@sub_group_tag')) > self.message_count[sub_group_tag]: + elif state == 'COINC_MSG' and len(cache_data.cache.query('sub_group==@sub_group_tag')) > _message_count[sub_group_tag]: # yet another pretty terminal output click.secho(f'SUB GROUP {sub_group_tag}:{"NEW COINCIDENT DETECTOR.. ".upper():^100}', bg='bright_green', fg='red') click.secho(f'{"Published an Alert!!!".upper():^100}', bg='bright_green', fg='red') @@ -546,6 +575,42 @@ def alert_decider(self): continue # ------------------------------------------------------------------------------------------------------------------ + def deal_with_the_cache(self, snews_message, is_test=False): + if not is_test: + which_cache_to_use = 'main' + self.coinc_data.add_to_cache(message=snews_message) + # run the search + self.alert_decider(which_cache_to_use) + # update message count + for sub_group_tag in self.coinc_data.cache['sub_group'].unique(): + self.message_count[sub_group_tag] = len( + self.coinc_data.cache.query('sub_group==@sub_group_tag')) + self.coinc_data.sub_group_state[sub_group_tag] = None + + self.coinc_data.updated = [] + # do not have a storage for the tests + if not is_test: + self.storage.insert_coinc_cache(self.coinc_data.cache) + sys.stdout.flush() + self.coinc_data.updated = [] + else: + which_cache_to_use = 'test' + self.test_coinc_data.add_to_cache(message=snews_message) + # run the search + self.alert_decider(which_cache_to_use) + # update message count + for sub_group_tag in self.test_coinc_data.cache['sub_group'].unique(): + self.test_message_count[sub_group_tag] = len( + self.test_coinc_data.cache.query('sub_group==@sub_group_tag')) + self.test_coinc_data.sub_group_state[sub_group_tag] = None + self.test_coinc_data.updated = [] + # do not have a storage for the tests + sys.stdout.flush() + + if self.show_table: + self.display_table(which_cache_to_use) ## don't display on the server + + #------------------------------------------------------------------------------------------------------------------- def run_coincidence(self): """ As the name states this method runs the coincidence system. @@ -586,22 +651,9 @@ def run_coincidence(self): terminal_output += click.style(f"\t>{snews_message['detector_name']}, {snews_message['received_time']}", fg='bright_blue') click.secho(terminal_output) # add to cache - self.coinc_data.add_to_cache(message=snews_message) - if self.show_table: - self.display_table() ## don't display on the server - self.alert_decider() - # update message count - for sub_group_tag in self.coinc_data.cache['sub_group'].unique(): - self.message_count[sub_group_tag] = len( - self.coinc_data.cache.query('sub_group==@sub_group_tag')) - self.coinc_data.sub_group_state[sub_group_tag] = None - - self.coinc_data.updated = [] - - self.storage.insert_coinc_cache(self.coinc_data.cache) - sys.stdout.flush() - - self.coinc_data.updated = [] + ### if actual observation, use coincidence cache, else if testing use test cache + + # for each read message reduce the retriable err count if self.retriable_error_count > 1: From bac76e50a1dc0353e6f23ce3c788d24c64c050f1 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:05:24 -0500 Subject: [PATCH 10/16] is test check --- snews_cs/snews_coinc.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/snews_cs/snews_coinc.py b/snews_cs/snews_coinc.py index 34e3d95..7cdfab3 100644 --- a/snews_cs/snews_coinc.py +++ b/snews_cs/snews_coinc.py @@ -575,7 +575,25 @@ def alert_decider(self, which_cache_to_use="main"): continue # ------------------------------------------------------------------------------------------------------------------ - def deal_with_the_cache(self, snews_message, is_test=False): + def deal_with_the_cache(self, snews_message): + """ Check if the message is a test or not, then add it to the cache and run the alert decider + + Parameters + ---------- + snews_message: dict read from the Kafka stream + + Returns + ------- + adds messages to cache and runs the coincidence decider + """ + if "meta" in snews_message.keys(): + is_test = snews_message['meta'].get('is_test', False) + else: + if "is_test" in snews_message.keys(): + is_test = snews_message['is_test'] + else: + is_test = False + if not is_test: which_cache_to_use = 'main' self.coinc_data.add_to_cache(message=snews_message) @@ -652,8 +670,7 @@ def run_coincidence(self): click.secho(terminal_output) # add to cache ### if actual observation, use coincidence cache, else if testing use test cache - - + self.deal_with_the_cache(snews_message) # for each read message reduce the retriable err count if self.retriable_error_count > 1: From 250bdd1877eda9d5a53b2837e0601e51e25f5449 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:06:36 -0500 Subject: [PATCH 11/16] is test check --- snews_cs/snews_coinc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snews_cs/snews_coinc.py b/snews_cs/snews_coinc.py index 7cdfab3..c77c61a 100644 --- a/snews_cs/snews_coinc.py +++ b/snews_cs/snews_coinc.py @@ -580,7 +580,7 @@ def deal_with_the_cache(self, snews_message): Parameters ---------- - snews_message: dict read from the Kafka stream + snews_message: dict read from the Kafka stream. Returns ------- From 30e7e7669cf83d10b51ab54c5138c7e365818a43 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:07:26 -0500 Subject: [PATCH 12/16] missing commit --- snews_cs/alert_pub.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/snews_cs/alert_pub.py b/snews_cs/alert_pub.py index 6391ca5..d8a912d 100644 --- a/snews_cs/alert_pub.py +++ b/snews_cs/alert_pub.py @@ -15,7 +15,7 @@ class AlertPublisher: """ Class to publish SNEWS SuperNova Alerts based on coincidence """ - def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True): + def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True, is_test=False): """ Alert publisher constructor Parameters @@ -36,6 +36,11 @@ def __init__(self, env_path=None, verbose=True, auth=True, firedrill_mode=True): self.alert_topic = os.getenv("ALERT_TOPIC") self.verbose = verbose + if is_test: + # use a test topic + self.alert_topic = os.getenv("CONNECTION_TEST_TOPIC") + + def __enter__(self): self.stream = Stream(until_eos=True, auth=self.auth).open(self.alert_topic, 'w') return self From 372a07486cbcec143bb07f19be3d04ae784e54e0 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:34:01 -0500 Subject: [PATCH 13/16] mention test in the alert type --- snews_cs/cs_alert_schema.py | 5 +++-- snews_cs/snews_coinc.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/snews_cs/cs_alert_schema.py b/snews_cs/cs_alert_schema.py index 1030568..4b9ed5e 100644 --- a/snews_cs/cs_alert_schema.py +++ b/snews_cs/cs_alert_schema.py @@ -34,7 +34,7 @@ def id_format(self, num_detectors): else: return f'SNEWS_Coincidence_ALERT-UPDATE {date_time}' - def get_cs_alert_schema(self, data): + def get_cs_alert_schema(self, data, which_cache_to_use='main'): """ Create a message schema for alert. Internally called in hop_pub @@ -50,8 +50,9 @@ def get_cs_alert_schema(self, data): """ id = self.id_format(len(data['detector_names'])) + alert_type = "TEST "+ data['alert_type'] if which_cache_to_use=='test' else data['alert_type'] return {"_id": id, - "alert_type":data['alert_type'], + "alert_type":alert_type, "server_tag": data['server_tag'], "False Alarm Prob": f"Would happen every {data['false_alarm_prob']:.2e} year", "detector_names": data['detector_names'], diff --git a/snews_cs/snews_coinc.py b/snews_cs/snews_coinc.py index c77c61a..d39836b 100644 --- a/snews_cs/snews_coinc.py +++ b/snews_cs/snews_coinc.py @@ -483,7 +483,7 @@ def send_alert(self, sub_group_tag, alert_type, which_cache_to_use="main"): with alert_publisher as pub: - alert = self.alert_schema.get_cs_alert_schema(data=alert_data) + alert = self.alert_schema.get_cs_alert_schema(data=alert_data, which_cache_to_use=which_cache_to_use) pub.send(alert) # only check to see if email or slack should be sent if the alert is not a test alert if which_cache_to_use == 'main': From f172b474663bdeceb9b36bb8ca1c0530fb17415b Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:41:11 -0500 Subject: [PATCH 14/16] temp fix --- snews_cs/snews_coinc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/snews_cs/snews_coinc.py b/snews_cs/snews_coinc.py index d39836b..f4d3881 100644 --- a/snews_cs/snews_coinc.py +++ b/snews_cs/snews_coinc.py @@ -461,7 +461,10 @@ def display_table(self, which_cache_to_use): def send_alert(self, sub_group_tag, alert_type, which_cache_to_use="main"): if which_cache_to_use == 'main': sub_df = self.coinc_data.cache.query('sub_group==@sub_group_tag') - false_alarm_prob = cache_false_alarm_rate(cache_sub_list=sub_df, hb_cache=self.heartbeat.cache_df) + try: + false_alarm_prob = cache_false_alarm_rate(cache_sub_list=sub_df, hb_cache=self.heartbeat.cache_df) + except: + false_alarm_prob = "(couldn't compute)" alert_publisher = self.alert else: sub_df = self.test_coinc_data.cache.query('sub_group==@sub_group_tag') From cc92ea14da6ebe58ec918e611fd28849926b7047 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:43:42 -0500 Subject: [PATCH 15/16] clean up --- logs/.gitignore | 1 - snews_cs/snews_coinc.py | 3 --- 2 files changed, 4 deletions(-) delete mode 100644 logs/.gitignore diff --git a/logs/.gitignore b/logs/.gitignore deleted file mode 100644 index 8b13789..0000000 --- a/logs/.gitignore +++ /dev/null @@ -1 +0,0 @@ - diff --git a/snews_cs/snews_coinc.py b/snews_cs/snews_coinc.py index f4d3881..6483e41 100644 --- a/snews_cs/snews_coinc.py +++ b/snews_cs/snews_coinc.py @@ -507,14 +507,11 @@ def alert_decider(self, which_cache_to_use="main"): """ # mkae a pretty terminal output click.secho(f'{"=" * 100}', fg='bright_red') - # loop through the sub group tag and state - # print(f'TEST {self.coinc_data.sub_group_state}') # decide which cache to use if which_cache_to_use == 'main': cache_data = self.coinc_data _message_count = self.message_count - # send_alert arg else: cache_data = self.test_coinc_data _message_count = self.test_message_count From 113eee3b3dd2157eaf9488463a1bdc3436fee1e8 Mon Sep 17 00:00:00 2001 From: KaraMelih Date: Thu, 15 Aug 2024 15:47:13 -0500 Subject: [PATCH 16/16] printing mistake --- snews_cs/cs_alert_schema.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snews_cs/cs_alert_schema.py b/snews_cs/cs_alert_schema.py index 4b9ed5e..13cb7dc 100644 --- a/snews_cs/cs_alert_schema.py +++ b/snews_cs/cs_alert_schema.py @@ -51,10 +51,14 @@ def get_cs_alert_schema(self, data, which_cache_to_use='main'): """ id = self.id_format(len(data['detector_names'])) alert_type = "TEST "+ data['alert_type'] if which_cache_to_use=='test' else data['alert_type'] + try: + far = f"Would happen every {data['false_alarm_prob']:.2e} year" + except: + far = data['false_alarm_prob'] return {"_id": id, "alert_type":alert_type, "server_tag": data['server_tag'], - "False Alarm Prob": f"Would happen every {data['false_alarm_prob']:.2e} year", + "False Alarm Prob": far, "detector_names": data['detector_names'], "sent_time": id.split(' ')[1], "p_values": data['p_vals'],