From 60000040846c40208db4812753a2c0598b815fbb Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Fri, 19 Apr 2024 11:00:49 +0200 Subject: [PATCH 1/2] Log info when a message is sent --- src/pytroll_watchers/publisher.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pytroll_watchers/publisher.py b/src/pytroll_watchers/publisher.py index ac42c26..2ec16c4 100644 --- a/src/pytroll_watchers/publisher.py +++ b/src/pytroll_watchers/publisher.py @@ -1,6 +1,7 @@ """Common functions for publishing messages.""" import datetime +import logging from contextlib import closing, suppress from copy import deepcopy @@ -8,6 +9,8 @@ from posttroll.publisher import create_publisher_from_dict_config from trollsift import parse +logger = logging.getLogger(__name__) + def file_publisher_from_generator(generator, publisher_config, message_config): """Publish files coming from local filesystem events. @@ -31,6 +34,7 @@ def file_publisher_from_generator(generator, publisher_config, message_config): apply_aliases(aliases, file_metadata) amended_message_config["data"].update(file_metadata) msg = Message(**amended_message_config) + logger.info(f"Sending {str(msg)}") publisher.send(str(msg)) From ed8bd3ae7b7407410b3854320000a43a38b1647a Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Fri, 19 Apr 2024 11:21:35 +0200 Subject: [PATCH 2/2] Fix fs publishing to be a dict, not a string --- src/pytroll_watchers/publisher.py | 3 ++- tests/test_bucket_notification_watcher.py | 4 ++-- tests/test_copernicus_dataspace_watcher.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pytroll_watchers/publisher.py b/src/pytroll_watchers/publisher.py index 2ec16c4..131cf53 100644 --- a/src/pytroll_watchers/publisher.py +++ b/src/pytroll_watchers/publisher.py @@ -1,6 +1,7 @@ """Common functions for publishing messages.""" import datetime +import json import logging from contextlib import closing, suppress from copy import deepcopy @@ -29,7 +30,7 @@ def file_publisher_from_generator(generator, publisher_config, message_config): amended_message_config = deepcopy(message_config) amended_message_config["data"]["uri"] = file_item.as_uri() with suppress(AttributeError): - amended_message_config["data"]["fs"] = file_item.fs.to_json() + amended_message_config["data"]["fs"] = json.loads(file_item.fs.to_json()) aliases = amended_message_config.pop("aliases", {}) apply_aliases(aliases, file_metadata) amended_message_config["data"].update(file_metadata) diff --git a/tests/test_bucket_notification_watcher.py b/tests/test_bucket_notification_watcher.py index 3fd7274..3045ec1 100644 --- a/tests/test_bucket_notification_watcher.py +++ b/tests/test_bucket_notification_watcher.py @@ -106,8 +106,8 @@ def test_publish_paths(patched_bucket_listener, caplog): # noqa message = Message(rawstr=messages[0]) assert message.data["uri"] == "s3://viirs-data/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5" assert message.data["sensor"] == "viirs" - assert message.data["fs"] == ('{"cls": "s3fs.core.S3FileSystem", "protocol": "s3", "args": [], ' - '"profile": "someprofile"}') + assert message.data["fs"] == {"cls": "s3fs.core.S3FileSystem", "protocol": "s3", "args": [], + "profile": "someprofile"} assert "Starting watch on 'viirs-data'" in caplog.text diff --git a/tests/test_copernicus_dataspace_watcher.py b/tests/test_copernicus_dataspace_watcher.py index 1def1d5..8587946 100644 --- a/tests/test_copernicus_dataspace_watcher.py +++ b/tests/test_copernicus_dataspace_watcher.py @@ -113,6 +113,6 @@ def test_publish_paths(caplog): message = Message(rawstr=messages[0]) assert message.data["uri"] == "s3:///eodata/Sentinel-3/OLCI/OL_1_EFR___/2024/04/15/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3" assert message.data["sensor"] == "olci" - assert message.data["fs"] == ('{"cls": "s3fs.core.S3FileSystem", "protocol": "s3", "args": [], ' - '"profile": "someprofile"}') + assert message.data["fs"] == {"cls": "s3fs.core.S3FileSystem", "protocol": "s3", "args": [], + "profile": "someprofile"} assert f"Starting watch on dataspace for '{filter_string}'" in caplog.text