From 38c820c50576d5f6a526c4fad75b04239d677c54 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 30 Apr 2024 11:55:20 +0200 Subject: [PATCH] Fix datastor mda handling --- src/pytroll_watchers/datastore_watcher.py | 14 ++++++++++++-- tests/test_datastore.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/pytroll_watchers/datastore_watcher.py b/src/pytroll_watchers/datastore_watcher.py index 7854cd4..4b25c65 100644 --- a/src/pytroll_watchers/datastore_watcher.py +++ b/src/pytroll_watchers/datastore_watcher.py @@ -92,8 +92,18 @@ def generate_download_links(search_params, ds_auth): path = UPath(links[0]["href"], encoded=True, client_kwargs=client_args) # In the future, it might be interesting to generate items from the sip-entries, as # they contain individual files for zip archives. - - yield path, feature + mda = dict() + mda["boundary"] = feature["geometry"] + acq_info = feature["properties"]["acquisitionInformation"][0] + mda["platform_name"] = acq_info["platform"]["platformShortName"] + mda["sensor"] = acq_info["instrument"]["instrumentShortName"].lower() + mda["orbit_number"] = acq_info["acquisitionParameters"]["orbitNumber"] + start_string, end_string = feature["properties"]["date"].split("/") + mda["start_time"] = datetime.datetime.fromisoformat(start_string) + mda["end_time"] = datetime.datetime.fromisoformat(end_string) + mda["product_type"] = str(collection) + mda["checksum"] = dict(algorithm="md5", hash=feature["properties"]["extraInformation"]["md5"]) + yield path, mda class DatastoreOAuth2Session(): diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 3cb62ea..d1744a8 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -48,7 +48,14 @@ def test_datastore_get_download_links_since(search_params): path, mda = features[0] assert str(path) == "https://api.eumetsat.int/data/download/1.0.0/collections/EO%3AEUM%3ADAT%3A0407/products/S3B_OL_2_WFR____20240416T104217_20240416T104517_20240417T182315_0180_092_051_1980_MAR_O_NT_003.SEN3" assert expected_token in path.storage_options["client_kwargs"]["headers"]["Authorization"] - assert "sip-entries" in mda["properties"]["links"] + assert mda["boundary"]["coordinates"][0][0] == [-14.3786, 52.4516] + assert mda["platform_name"] == "Sentinel-3B" + assert mda["sensor"] == "olci" + assert mda["start_time"] == datetime.datetime(2024, 4, 16, 10, 42, 16, 954262, tzinfo=datetime.timezone.utc) + assert mda["end_time"] == datetime.datetime(2024, 4, 16, 10, 45, 16, 954262, tzinfo=datetime.timezone.utc) + assert mda["orbit_number"] == 31123 + assert mda["product_type"] == "EO:EUM:DAT:0407" + assert mda["checksum"] == dict(algorithm="md5", hash="9057eb08f2a4e9f4c5a8d2eeaacedaef") @contextmanager @@ -98,7 +105,6 @@ def test_datastore_file_generator(tmp_path, search_params): path, mda = features[0] assert str(path) == "https://api.eumetsat.int/data/download/1.0.0/collections/EO%3AEUM%3ADAT%3A0407/products/S3B_OL_2_WFR____20240416T104217_20240416T104517_20240417T182315_0180_092_051_1980_MAR_O_NT_003.SEN3" assert expected_token in path.storage_options["client_kwargs"]["headers"]["Authorization"] - assert "sip-entries" in mda["properties"]["links"] @freeze_time(datetime.datetime.now(datetime.timezone.utc))