From c6876e4091ba3167565a874981b497034211bd32 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 29 Apr 2024 13:44:17 +0200 Subject: [PATCH 01/14] Add a watcher for DHuS instances --- docs/source/index.rst | 13 ++- pyproject.toml | 3 +- src/pytroll_watchers/dhus_watcher.py | 150 +++++++++++++++++++++++++ src/pytroll_watchers/main_interface.py | 13 ++- src/pytroll_watchers/publisher.py | 1 + tests/dhus_responses.yaml | 133 ++++++++++++++++++++++ tests/test_dhus.py | 67 +++++++++++ 7 files changed, 372 insertions(+), 8 deletions(-) create mode 100644 src/pytroll_watchers/dhus_watcher.py create mode 100644 tests/dhus_responses.yaml create mode 100644 tests/test_dhus.py diff --git a/docs/source/index.rst b/docs/source/index.rst index 6221078..95b2596 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -122,10 +122,21 @@ Minio bucket notification watcher :members: Copernicus dataspace watcher ---------------------------------- +---------------------------- .. automodule:: pytroll_watchers.dataspace_watcher :members: +EUMETSAT datastore watcher +-------------------------- +.. automodule:: pytroll_watchers.datastore_watcher + :members: + +DHuS watcher +------------ +.. automodule:: pytroll_watchers.dhus_watcher + :members: + + Testing utilities ----------------- .. automodule:: pytroll_watchers.testing diff --git a/pyproject.toml b/pyproject.toml index e4ebd05..3d5e2c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Utility functions and scripts to watch for new files on local or authors = [ { name = "Martin Raspaud", email = "martin.raspaud@smhi.se" } ] -dependencies = ["universal-pathlib", "trollsift", "pyyaml"] +dependencies = ["universal-pathlib", "trollsift", "pyyaml", "geojson"] readme = "README.md" requires-python = ">= 3.10" license = {file = "LICENSE.txt"} @@ -22,6 +22,7 @@ local = ["watchdog"] publishing = ["posttroll"] ssh = ["paramiko"] dataspace = ["oauthlib", "requests_oauthlib", "s3fs"] +dhus = ["defusedxml"] [build-system] requires = ["hatchling", "hatch-vcs"] diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py new file mode 100644 index 0000000..c738fb1 --- /dev/null +++ b/src/pytroll_watchers/dhus_watcher.py @@ -0,0 +1,150 @@ +"""Watcher for DHuS instances. + +For more information about DHuS, check out +https://sentineldatahub.github.io/DataHubSystem/about.html +""" + +import datetime as dt +import logging +from contextlib import suppress +from urllib.parse import urljoin + +import requests +from geojson import Polygon +from upath import UPath + +from pytroll_watchers.dataspace_watcher import run_every +from pytroll_watchers.publisher import file_publisher_from_generator + +logger = logging.getLogger(__name__) + + +def file_publisher(fs_config, publisher_config, message_config): + """Publish files coming from local filesystem events. + + Args: + fs_config: the configuration for the filesystem watching, will be passed as argument to `file_generator`. + publisher_config: The configuration dictionary to pass to the posttroll publishing functions. + message_config: The information needed to complete the posttroll message generation. Will be amended + with the file metadata, and passed directly to posttroll's Message constructor. + """ + logger.info(f"Starting watch on dhus for '{fs_config['filter_params']}'") + generator = file_generator(**fs_config) + return file_publisher_from_generator(generator, publisher_config, message_config) + + +def file_generator(server, filter_params, polling_interval, start_from=None): + """Generate new objects by polling a DHuS instance. + + Args: + server: the DHuS server to use. + filter_params: the list of filter parameters to use for narrowing the data to poll. For example, to poll IW sar + data, it can be `substringof('IW_GRDH',Name)`. For more information of the filter parameters, check: + https://scihub.copernicus.eu/twiki/do/view/SciHubUserGuide/ODataAPI#filter + polling_interval: the interval (timedelta object or kwargs to timedelta) at which the DHUS will be polled. + start_from: how far back in time to fetch the data the first time. This is helpful for the first iteration of + the generator, so that data from the past can be fetched, to fill a possible gap. Default to 0, meaning + nothing older than when the generator starts will be fetched. Same format accepted as polling_interval. + + Yields: + Tuples of UPath (http) and metadata. + + + Note: + As this watcher uses requests, the authentication information should be stored in a .netrc file. + """ + with suppress(TypeError): + polling_interval = dt.timedelta(**polling_interval) + with suppress(TypeError): + start_from = dt.timedelta(**start_from) + if start_from is None: + start_from = dt.timedelta(0) + + last_pub_date = dt.datetime.now(dt.timezone.utc) - start_from + for next_check in run_every(polling_interval): + generator = generate_download_links_since(server, filter_params, last_pub_date) + for path, metadata in generator: + last_pub_date = update_last_publication_date(last_pub_date, metadata) + yield path, metadata + logger.info("Finished polling.") + if next_check > dt.datetime.now(dt.timezone.utc): + logger.info(f"Next iteration at {next_check}") + + +def update_last_publication_date(last_publication_date, metadata): + """Update the last publication data based on the metadata.""" + publication_date = metadata.pop("ingestion_date") + if publication_date > last_publication_date: + last_publication_date = publication_date + return last_publication_date + + +def generate_download_links_since(server, filter_params, last_publication_date): + """Generate download links for the data published since `last_publication_date`.""" + last_publication_date = last_publication_date.astimezone(dt.timezone.utc) + # remove timezone info as dhus considers all times utc + pub_string = last_publication_date.isoformat(timespec="milliseconds")[:-6] + filter_params = filter_params + [f"IngestionDate gt datetime'{pub_string}'"] + yield from generate_download_links(server, filter_params) + + +def generate_download_links(server, filter_params): + """Generate download links. + + The filter params we can use are defined here: https://scihub.copernicus.eu/twiki/do/view/SciHubUserGuide/ODataAPI#filter + """ + filter_string = " and ".join(filter_params) + + url = urljoin(server, f"/odata/v1/Products?$format=json&$filter={filter_string}&$expand=Attributes") + + response = requests.get(url, timeout=60) + response.raise_for_status() + + entries = response.json() + + for entry in entries["d"]["results"]: + mda = dict() + path = UPath(entry["__metadata"]["media_src"]) + mda["boundary"] = extract_boundary(entry) + results_dict = construct_results_dict(entry) + mda["platform_name"] = results_dict["Satellite name"].capitalize() + results_dict["Satellite number"] + mda["sensor"] = results_dict["Instrument"] + mda["ingestion_date"] = dt.datetime.fromisoformat(results_dict["Ingestion Date"]) + mda["product_type"] = results_dict["Product type"] + mda["start_time"] = dt.datetime.fromisoformat(results_dict["Sensing start"]) + mda["end_time"] = dt.datetime.fromisoformat(results_dict["Sensing stop"]) + mda["orbit_number"] = int(results_dict["Orbit number (start)"]) + + mda["checksum"] = dict(algorithm=entry["Checksum"]["Algorithm"], hash=entry["Checksum"]["Value"]) + mda["size"] = int(entry["ContentLength"]) + yield path, mda + +def construct_results_dict(entry): + """Construct a dict from then "results" item in entry.""" + results = entry["Attributes"]["results"] + results_dict = {result["Id"]: result["Value"] for result in results} + return results_dict + + +def extract_boundary(entry): + """Extract the boundary from the entry metadata.""" + gml, nsmap = read_gml(entry["ContentGeometry"]) + boundary_text = gml.find("gml:outerBoundaryIs/gml:LinearRing/gml:coordinates", namespaces=nsmap).text + boundary_list = (coords.split(",") for coords in boundary_text.strip().split(" ")) + boundary = Polygon([(float(lon), float(lat)) for (lat, lon) in boundary_list]) + return boundary + +def read_gml(gml_string): + """Read the gml string.""" + from xml.etree import ElementTree + + from defusedxml.ElementTree import DefusedXMLParser + parser = ElementTree.XMLPullParser(["start-ns", "end"], _parser=DefusedXMLParser()) + parser.feed(gml_string) + + nsmap = dict() + + for event, elem in parser.read_events(): + if event == "start-ns": + nsmap[elem[0]] = elem[1] + return elem, nsmap diff --git a/src/pytroll_watchers/main_interface.py b/src/pytroll_watchers/main_interface.py index bc04110..2d97c0c 100644 --- a/src/pytroll_watchers/main_interface.py +++ b/src/pytroll_watchers/main_interface.py @@ -5,6 +5,7 @@ import yaml +from pytroll_watchers.dhus_watcher import file_publisher as dhus_publisher from pytroll_watchers.local_watcher import file_generator as local_generator from pytroll_watchers.local_watcher import file_publisher as local_publisher from pytroll_watchers.minio_notification_watcher import file_generator as minio_generator @@ -25,6 +26,8 @@ def get_publisher_for_backend(backend): return minio_publisher elif backend == "local": return local_publisher + elif backend == "dhus": + return dhus_publisher else: raise ValueError(f"Unknown backend {backend}.") @@ -54,12 +57,10 @@ def publish_from_config(config): config: a dictionary containing the `backend` string (`local` or `minio`), and `fs_config`, `publisher_config` and `message_config` dictionaries. """ - if config["backend"] == "local": - return local_publisher(config["fs_config"], config["publisher_config"], config["message_config"]) - elif config["backend"] == "minio": - return minio_publisher(config["fs_config"], config["publisher_config"], config["message_config"]) - else: - raise ValueError(f"Unknown backend {config['backend']}") + backend = config["backend"] + publisher = get_publisher_for_backend(backend) + return publisher(config["fs_config"], config["publisher_config"], config["message_config"]) + def cli(args=None): diff --git a/src/pytroll_watchers/publisher.py b/src/pytroll_watchers/publisher.py index caf13a0..0d8beba 100644 --- a/src/pytroll_watchers/publisher.py +++ b/src/pytroll_watchers/publisher.py @@ -46,6 +46,7 @@ def file_publisher_from_generator(generator, publisher_config, message_config): with closing(publisher): for file_item, file_metadata in generator: amended_message_config = deepcopy(message_config) + amended_message_config.setdefault("data", {}) amended_message_config["data"]["uri"] = file_item.as_uri() amended_message_config["data"]["uid"] = file_item.name with suppress(AttributeError): diff --git a/tests/dhus_responses.yaml b/tests/dhus_responses.yaml new file mode 100644 index 0000000..5b1e02f --- /dev/null +++ b/tests/dhus_responses.yaml @@ -0,0 +1,133 @@ +responses: +- response: + auto_calculate_content_length: false + body: '{"d":{"results":[{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')","uri":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')","type":"DHuS.Product","content_type":"application/octet-stream","media_src":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')/$value","edit_media":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')/$value"},"Id":"e49f8de5-1647-4a7b-ba69-268f9aa77f42","Name":"S1A_IW_GRDH_1SDV_20240429T064623_20240429T064648_053645_0683C0_54F4","ContentType":"application/octet-stream","ContentLength":"1774829329","ChildrenNumber":"2","Value":null,"CreationDate":"\/Date(1714376943458)\/","IngestionDate":"\/Date(1714375999009)\/","ModificationDate":"\/Date(1714376943458)\/","EvictionDate":"\/Date(1716968943458)\/","Online":true,"OnDemand":false,"ContentDate":{"__metadata":{"type":"DHuS.TimeRange"},"Start":"\/Date(1714373183104)\/","End":"\/Date(1714373208102)\/"},"Checksum":{"__metadata":{"type":"DHuS.Checksum"},"Algorithm":"MD5","Value":"bf5adf26050693a7515fa8197b9d89a8"},"ContentGeometry":" 57.399067,-4.545108 + 58.883202,-3.913711 59.317738,-8.45609 57.826885,-8.902448 57.399067,-4.545108 ","Products":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')/Products"}},"Nodes":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')/Nodes"}},"Attributes":{"results":[{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","type":"DHuS.Attribute"},"Id":"Instrument + mode","Name":"Instrument mode","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","type":"DHuS.Attribute"},"Id":"Footprint","Name":"Footprint","ContentType":"text/plain","ContentLength":"308","Value":" 57.399067,-4.545108 + 58.883202,-3.913711 59.317738,-8.45609 57.826885,-8.902448 57.399067,-4.545108 ","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","type":"DHuS.Attribute"},"Id":"JTS + footprint","Name":"JTS footprint","ContentType":"text/plain","ContentLength":"114","Value":"POLYGON + ((-4.545108 57.399067, -3.913711 58.883202, -8.45609 59.317738, -8.902448 57.826885, + -4.545108 57.399067))","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Orbit + number (start)","Name":"Orbit number (start)","ContentType":"text/plain","ContentLength":"5","Value":"53645","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Relative + orbit (start)","Name":"Relative orbit (start)","ContentType":"text/plain","ContentLength":"2","Value":"23","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","type":"DHuS.Attribute"},"Id":"Format","Name":"Format","ContentType":"text/plain","ContentLength":"0","Value":"","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","type":"DHuS.Attribute"},"Id":"Sensing + start","Name":"Sensing start","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:46:23.104Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","type":"DHuS.Attribute"},"Id":"Sensing + stop","Name":"Sensing stop","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:46:48.102Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","type":"DHuS.Attribute"},"Id":"Satellite + name","Name":"Satellite name","ContentType":"text/plain","ContentLength":"10","Value":"SENTINEL-1","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","type":"DHuS.Attribute"},"Id":"Satellite + number","Name":"Satellite number","ContentType":"text/plain","ContentLength":"1","Value":"A","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","type":"DHuS.Attribute"},"Id":"Filename","Name":"Filename","ContentType":"text/plain","ContentLength":"72","Value":"S1A_IW_GRDH_1SDV_20240429T064623_20240429T064648_053645_0683C0_54F4.SAFE","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","type":"DHuS.Attribute"},"Id":"Instrument","Name":"Instrument","ContentType":"text/plain","ContentLength":"3","Value":"SAR","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","type":"DHuS.Attribute"},"Id":"Date","Name":"Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:46:23.104Z","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","type":"DHuS.Attribute"},"Id":"Size","Name":"Size","ContentType":"text/plain","ContentLength":"7","Value":"1.65 + GB","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","type":"DHuS.Attribute"},"Id":"Product + type","Name":"Product type","ContentType":"text/plain","ContentLength":"10","Value":"IW_GRDH_1S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","type":"DHuS.Attribute"},"Id":"Pass + direction","Name":"Pass direction","ContentType":"text/plain","ContentLength":"10","Value":"DESCENDING","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","type":"DHuS.Attribute"},"Id":"Ingestion + Date","Name":"Ingestion Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T07:33:19.009Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","type":"DHuS.Attribute"},"Id":"Identifier","Name":"Identifier","ContentType":"text/plain","ContentLength":"67","Value":"S1A_IW_GRDH_1SDV_20240429T064623_20240429T064648_053645_0683C0_54F4","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","type":"DHuS.Attribute"},"Id":"Timeliness + Category","Name":"Timeliness Category","ContentType":"text/plain","ContentLength":"6","Value":"NRT-3h","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","type":"DHuS.Attribute"},"Id":"Instrument + swath","Name":"Instrument swath","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","type":"DHuS.Attribute"},"Id":"Mission + datatake id","Name":"Mission datatake id","ContentType":"text/plain","ContentLength":"6","Value":"426944","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","type":"DHuS.Attribute"},"Id":"Product + class","Name":"Product class","ContentType":"text/plain","ContentLength":"1","Value":"S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","type":"DHuS.Attribute"},"Id":"Product + composition","Name":"Product composition","ContentType":"text/plain","ContentLength":"5","Value":"Slice","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","type":"DHuS.Attribute"},"Id":"Cycle + number","Name":"Cycle number","ContentType":"text/plain","ContentLength":"3","Value":"321","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","type":"DHuS.Attribute"},"Id":"Polarisation","Name":"Polarisation","ContentType":"text/plain","ContentLength":"5","Value":"VV&VH","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","type":"DHuS.Attribute"},"Id":"Slice + number","Name":"Slice number","ContentType":"text/plain","ContentLength":"1","Value":"5","Category":"product"}]},"Class":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''e49f8de5-1647-4a7b-ba69-268f9aa77f42'')/Class"}}},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')","uri":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')","type":"DHuS.Product","content_type":"application/octet-stream","media_src":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')/$value","edit_media":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')/$value"},"Id":"162f165a-23c3-4212-869e-4f38b9dade63","Name":"S1A_IW_GRDH_1SDV_20240429T065238_20240429T065303_053645_0683C0_D62B","ContentType":"application/octet-stream","ContentLength":"1763992817","ChildrenNumber":"2","Value":null,"CreationDate":"\/Date(1714376943746)\/","IngestionDate":"\/Date(1714375996339)\/","ModificationDate":"\/Date(1714376943746)\/","EvictionDate":"\/Date(1716968943746)\/","Online":true,"OnDemand":false,"ContentDate":{"__metadata":{"type":"DHuS.TimeRange"},"Start":"\/Date(1714373558105)\/","End":"\/Date(1714373583103)\/"},"Checksum":{"__metadata":{"type":"DHuS.Checksum"},"Algorithm":"MD5","Value":"467c519f04199e209d5ea2ba644e8534"},"ContentGeometry":" 34.945248,-11.227338 + 36.449383,-10.868676 36.860729,-13.766738 35.358513,-14.070704 34.945248,-11.227338 ","Products":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')/Products"}},"Nodes":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')/Nodes"}},"Attributes":{"results":[{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","type":"DHuS.Attribute"},"Id":"Instrument + mode","Name":"Instrument mode","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","type":"DHuS.Attribute"},"Id":"Footprint","Name":"Footprint","ContentType":"text/plain","ContentLength":"314","Value":" 34.945248,-11.227338 + 36.449383,-10.868676 36.860729,-13.766738 35.358513,-14.070704 34.945248,-11.227338 ","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","type":"DHuS.Attribute"},"Id":"JTS + footprint","Name":"JTS footprint","ContentType":"text/plain","ContentLength":"120","Value":"POLYGON + ((-11.227338 34.945248, -10.868676 36.449383, -13.766738 36.860729, -14.070704 + 35.358513, -11.227338 34.945248))","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Orbit + number (start)","Name":"Orbit number (start)","ContentType":"text/plain","ContentLength":"5","Value":"53645","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Relative + orbit (start)","Name":"Relative orbit (start)","ContentType":"text/plain","ContentLength":"2","Value":"23","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","type":"DHuS.Attribute"},"Id":"Format","Name":"Format","ContentType":"text/plain","ContentLength":"0","Value":"","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","type":"DHuS.Attribute"},"Id":"Sensing + start","Name":"Sensing start","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:52:38.105Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","type":"DHuS.Attribute"},"Id":"Sensing + stop","Name":"Sensing stop","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:53:03.103Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","type":"DHuS.Attribute"},"Id":"Satellite + name","Name":"Satellite name","ContentType":"text/plain","ContentLength":"10","Value":"SENTINEL-1","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","type":"DHuS.Attribute"},"Id":"Satellite + number","Name":"Satellite number","ContentType":"text/plain","ContentLength":"1","Value":"A","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","type":"DHuS.Attribute"},"Id":"Filename","Name":"Filename","ContentType":"text/plain","ContentLength":"72","Value":"S1A_IW_GRDH_1SDV_20240429T065238_20240429T065303_053645_0683C0_D62B.SAFE","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","type":"DHuS.Attribute"},"Id":"Instrument","Name":"Instrument","ContentType":"text/plain","ContentLength":"3","Value":"SAR","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","type":"DHuS.Attribute"},"Id":"Date","Name":"Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:52:38.105Z","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","type":"DHuS.Attribute"},"Id":"Size","Name":"Size","ContentType":"text/plain","ContentLength":"7","Value":"1.64 + GB","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","type":"DHuS.Attribute"},"Id":"Product + type","Name":"Product type","ContentType":"text/plain","ContentLength":"10","Value":"IW_GRDH_1S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","type":"DHuS.Attribute"},"Id":"Pass + direction","Name":"Pass direction","ContentType":"text/plain","ContentLength":"10","Value":"DESCENDING","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","type":"DHuS.Attribute"},"Id":"Ingestion + Date","Name":"Ingestion Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T07:33:16.339Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","type":"DHuS.Attribute"},"Id":"Identifier","Name":"Identifier","ContentType":"text/plain","ContentLength":"67","Value":"S1A_IW_GRDH_1SDV_20240429T065238_20240429T065303_053645_0683C0_D62B","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","type":"DHuS.Attribute"},"Id":"Timeliness + Category","Name":"Timeliness Category","ContentType":"text/plain","ContentLength":"6","Value":"NRT-3h","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","type":"DHuS.Attribute"},"Id":"Instrument + swath","Name":"Instrument swath","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","type":"DHuS.Attribute"},"Id":"Mission + datatake id","Name":"Mission datatake id","ContentType":"text/plain","ContentLength":"6","Value":"426944","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","type":"DHuS.Attribute"},"Id":"Product + class","Name":"Product class","ContentType":"text/plain","ContentLength":"1","Value":"S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","type":"DHuS.Attribute"},"Id":"Product + composition","Name":"Product composition","ContentType":"text/plain","ContentLength":"5","Value":"Slice","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","type":"DHuS.Attribute"},"Id":"Cycle + number","Name":"Cycle number","ContentType":"text/plain","ContentLength":"3","Value":"321","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","type":"DHuS.Attribute"},"Id":"Polarisation","Name":"Polarisation","ContentType":"text/plain","ContentLength":"5","Value":"VV&VH","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","type":"DHuS.Attribute"},"Id":"Slice + number","Name":"Slice number","ContentType":"text/plain","ContentLength":"2","Value":"20","Category":"product"}]},"Class":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''162f165a-23c3-4212-869e-4f38b9dade63'')/Class"}}},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')","uri":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')","type":"DHuS.Product","content_type":"application/octet-stream","media_src":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')/$value","edit_media":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')/$value"},"Id":"d43bd5e3-2322-4b27-84d2-6734093f355f","Name":"S1A_IW_GRDH_1SDV_20240429T065303_20240429T065328_053645_0683C0_FD45","ContentType":"application/octet-stream","ContentLength":"1763308281","ChildrenNumber":"2","Value":null,"CreationDate":"\/Date(1714376943898)\/","IngestionDate":"\/Date(1714375994641)\/","ModificationDate":"\/Date(1714376943898)\/","EvictionDate":"\/Date(1716968943898)\/","Online":true,"OnDemand":false,"ContentDate":{"__metadata":{"type":"DHuS.TimeRange"},"Start":"\/Date(1714373583105)\/","End":"\/Date(1714373608103)\/"},"Checksum":{"__metadata":{"type":"DHuS.Checksum"},"Algorithm":"MD5","Value":"e42d02723bf0e7b0f4a3d42e0ebe5717"},"ContentGeometry":" 33.44025,-11.579451 + 34.94516,-11.227359 35.358295,-14.069745 33.855522,-14.370993 33.44025,-11.579451 ","Products":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')/Products"}},"Nodes":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')/Nodes"}},"Attributes":{"results":[{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","type":"DHuS.Attribute"},"Id":"Instrument + mode","Name":"Instrument mode","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","type":"DHuS.Attribute"},"Id":"Footprint","Name":"Footprint","ContentType":"text/plain","ContentLength":"311","Value":" 33.44025,-11.579451 + 34.94516,-11.227359 35.358295,-14.069745 33.855522,-14.370993 33.44025,-11.579451 ","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","type":"DHuS.Attribute"},"Id":"JTS + footprint","Name":"JTS footprint","ContentType":"text/plain","ContentLength":"117","Value":"POLYGON + ((-11.579451 33.44025, -11.227359 34.94516, -14.069745 35.358295, -14.370993 + 33.855522, -11.579451 33.44025))","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Orbit + number (start)","Name":"Orbit number (start)","ContentType":"text/plain","ContentLength":"5","Value":"53645","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Relative + orbit (start)","Name":"Relative orbit (start)","ContentType":"text/plain","ContentLength":"2","Value":"23","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","type":"DHuS.Attribute"},"Id":"Format","Name":"Format","ContentType":"text/plain","ContentLength":"0","Value":"","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","type":"DHuS.Attribute"},"Id":"Sensing + start","Name":"Sensing start","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:53:03.105Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","type":"DHuS.Attribute"},"Id":"Sensing + stop","Name":"Sensing stop","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:53:28.103Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","type":"DHuS.Attribute"},"Id":"Satellite + name","Name":"Satellite name","ContentType":"text/plain","ContentLength":"10","Value":"SENTINEL-1","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","type":"DHuS.Attribute"},"Id":"Satellite + number","Name":"Satellite number","ContentType":"text/plain","ContentLength":"1","Value":"A","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","type":"DHuS.Attribute"},"Id":"Filename","Name":"Filename","ContentType":"text/plain","ContentLength":"72","Value":"S1A_IW_GRDH_1SDV_20240429T065303_20240429T065328_053645_0683C0_FD45.SAFE","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","type":"DHuS.Attribute"},"Id":"Instrument","Name":"Instrument","ContentType":"text/plain","ContentLength":"3","Value":"SAR","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","type":"DHuS.Attribute"},"Id":"Date","Name":"Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:53:03.105Z","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","type":"DHuS.Attribute"},"Id":"Size","Name":"Size","ContentType":"text/plain","ContentLength":"7","Value":"1.64 + GB","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","type":"DHuS.Attribute"},"Id":"Product + type","Name":"Product type","ContentType":"text/plain","ContentLength":"10","Value":"IW_GRDH_1S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","type":"DHuS.Attribute"},"Id":"Pass + direction","Name":"Pass direction","ContentType":"text/plain","ContentLength":"10","Value":"DESCENDING","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","type":"DHuS.Attribute"},"Id":"Ingestion + Date","Name":"Ingestion Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T07:33:14.641Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","type":"DHuS.Attribute"},"Id":"Identifier","Name":"Identifier","ContentType":"text/plain","ContentLength":"67","Value":"S1A_IW_GRDH_1SDV_20240429T065303_20240429T065328_053645_0683C0_FD45","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","type":"DHuS.Attribute"},"Id":"Timeliness + Category","Name":"Timeliness Category","ContentType":"text/plain","ContentLength":"6","Value":"NRT-3h","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","type":"DHuS.Attribute"},"Id":"Instrument + swath","Name":"Instrument swath","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","type":"DHuS.Attribute"},"Id":"Mission + datatake id","Name":"Mission datatake id","ContentType":"text/plain","ContentLength":"6","Value":"426944","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","type":"DHuS.Attribute"},"Id":"Product + class","Name":"Product class","ContentType":"text/plain","ContentLength":"1","Value":"S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","type":"DHuS.Attribute"},"Id":"Product + composition","Name":"Product composition","ContentType":"text/plain","ContentLength":"5","Value":"Slice","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","type":"DHuS.Attribute"},"Id":"Cycle + number","Name":"Cycle number","ContentType":"text/plain","ContentLength":"3","Value":"321","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","type":"DHuS.Attribute"},"Id":"Polarisation","Name":"Polarisation","ContentType":"text/plain","ContentLength":"5","Value":"VV&VH","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","type":"DHuS.Attribute"},"Id":"Slice + number","Name":"Slice number","ContentType":"text/plain","ContentLength":"2","Value":"21","Category":"product"}]},"Class":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''d43bd5e3-2322-4b27-84d2-6734093f355f'')/Class"}}},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')","uri":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')","type":"DHuS.Product","content_type":"application/octet-stream","media_src":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')/$value","edit_media":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')/$value"},"Id":"a6b214dd-fe39-4f50-aa94-f84d8510cedc","Name":"S1A_IW_GRDH_1SDV_20240429T065418_20240429T065443_053645_0683C0_8D1D","ContentType":"application/octet-stream","ContentLength":"1761565015","ChildrenNumber":"2","Value":null,"CreationDate":"\/Date(1714376944004)\/","IngestionDate":"\/Date(1714375996964)\/","ModificationDate":"\/Date(1714376944004)\/","EvictionDate":"\/Date(1716968944004)\/","Online":true,"OnDemand":false,"ContentDate":{"__metadata":{"type":"DHuS.TimeRange"},"Start":"\/Date(1714373658105)\/","End":"\/Date(1714373683103)\/"},"Checksum":{"__metadata":{"type":"DHuS.Checksum"},"Algorithm":"MD5","Value":"9c251cbcaf8b5788cd1b440d7f7fd78b"},"ContentGeometry":" 28.92104,-12.602908 + 30.427998,-12.266554 30.84774,-14.964193 29.343462,-15.259768 28.92104,-12.602908 ","Products":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')/Products"}},"Nodes":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')/Nodes"}},"Attributes":{"results":[{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","type":"DHuS.Attribute"},"Id":"Instrument + mode","Name":"Instrument mode","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","type":"DHuS.Attribute"},"Id":"Footprint","Name":"Footprint","ContentType":"text/plain","ContentLength":"311","Value":" 28.92104,-12.602908 + 30.427998,-12.266554 30.84774,-14.964193 29.343462,-15.259768 28.92104,-12.602908 ","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","type":"DHuS.Attribute"},"Id":"JTS + footprint","Name":"JTS footprint","ContentType":"text/plain","ContentLength":"117","Value":"POLYGON + ((-12.602908 28.92104, -12.266554 30.427998, -14.964193 30.84774, -15.259768 + 29.343462, -12.602908 28.92104))","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Orbit + number (start)","Name":"Orbit number (start)","ContentType":"text/plain","ContentLength":"5","Value":"53645","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Relative + orbit (start)","Name":"Relative orbit (start)","ContentType":"text/plain","ContentLength":"2","Value":"23","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","type":"DHuS.Attribute"},"Id":"Format","Name":"Format","ContentType":"text/plain","ContentLength":"0","Value":"","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","type":"DHuS.Attribute"},"Id":"Sensing + start","Name":"Sensing start","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:54:18.105Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","type":"DHuS.Attribute"},"Id":"Sensing + stop","Name":"Sensing stop","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:54:43.103Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","type":"DHuS.Attribute"},"Id":"Satellite + name","Name":"Satellite name","ContentType":"text/plain","ContentLength":"10","Value":"SENTINEL-1","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","type":"DHuS.Attribute"},"Id":"Satellite + number","Name":"Satellite number","ContentType":"text/plain","ContentLength":"1","Value":"A","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","type":"DHuS.Attribute"},"Id":"Filename","Name":"Filename","ContentType":"text/plain","ContentLength":"72","Value":"S1A_IW_GRDH_1SDV_20240429T065418_20240429T065443_053645_0683C0_8D1D.SAFE","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","type":"DHuS.Attribute"},"Id":"Instrument","Name":"Instrument","ContentType":"text/plain","ContentLength":"3","Value":"SAR","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","type":"DHuS.Attribute"},"Id":"Date","Name":"Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:54:18.105Z","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","type":"DHuS.Attribute"},"Id":"Size","Name":"Size","ContentType":"text/plain","ContentLength":"7","Value":"1.64 + GB","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","type":"DHuS.Attribute"},"Id":"Product + type","Name":"Product type","ContentType":"text/plain","ContentLength":"10","Value":"IW_GRDH_1S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","type":"DHuS.Attribute"},"Id":"Pass + direction","Name":"Pass direction","ContentType":"text/plain","ContentLength":"10","Value":"DESCENDING","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","type":"DHuS.Attribute"},"Id":"Ingestion + Date","Name":"Ingestion Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T07:33:16.964Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","type":"DHuS.Attribute"},"Id":"Identifier","Name":"Identifier","ContentType":"text/plain","ContentLength":"67","Value":"S1A_IW_GRDH_1SDV_20240429T065418_20240429T065443_053645_0683C0_8D1D","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","type":"DHuS.Attribute"},"Id":"Timeliness + Category","Name":"Timeliness Category","ContentType":"text/plain","ContentLength":"6","Value":"NRT-3h","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","type":"DHuS.Attribute"},"Id":"Instrument + swath","Name":"Instrument swath","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","type":"DHuS.Attribute"},"Id":"Mission + datatake id","Name":"Mission datatake id","ContentType":"text/plain","ContentLength":"6","Value":"426944","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","type":"DHuS.Attribute"},"Id":"Product + class","Name":"Product class","ContentType":"text/plain","ContentLength":"1","Value":"S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","type":"DHuS.Attribute"},"Id":"Product + composition","Name":"Product composition","ContentType":"text/plain","ContentLength":"5","Value":"Slice","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","type":"DHuS.Attribute"},"Id":"Cycle + number","Name":"Cycle number","ContentType":"text/plain","ContentLength":"3","Value":"321","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","type":"DHuS.Attribute"},"Id":"Polarisation","Name":"Polarisation","ContentType":"text/plain","ContentLength":"5","Value":"VV&VH","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","type":"DHuS.Attribute"},"Id":"Slice + number","Name":"Slice number","ContentType":"text/plain","ContentLength":"2","Value":"24","Category":"product"}]},"Class":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''a6b214dd-fe39-4f50-aa94-f84d8510cedc'')/Class"}}},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')","uri":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')","type":"DHuS.Product","content_type":"application/octet-stream","media_src":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')/$value","edit_media":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')/$value"},"Id":"82aa1c63-6ef7-40e3-ba1a-40fe80e4d808","Name":"S1A_IW_GRDH_1SDV_20240429T065443_20240429T065508_053645_0683C0_FAB4","ContentType":"application/octet-stream","ContentLength":"1761134591","ChildrenNumber":"2","Value":null,"CreationDate":"\/Date(1714376943620)\/","IngestionDate":"\/Date(1714375995552)\/","ModificationDate":"\/Date(1714376943620)\/","EvictionDate":"\/Date(1716968943620)\/","Online":true,"OnDemand":false,"ContentDate":{"__metadata":{"type":"DHuS.TimeRange"},"Start":"\/Date(1714373683105)\/","End":"\/Date(1714373708103)\/"},"Checksum":{"__metadata":{"type":"DHuS.Checksum"},"Algorithm":"MD5","Value":"5489224cea04f4dd0c178dc294d0ea9d"},"ContentGeometry":" 27.413303,-12.934627 + 28.92095,-12.602928 29.343267,-15.259075 27.838463,-15.552944 27.413303,-12.934627 ","Products":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')/Products"}},"Nodes":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')/Nodes"}},"Attributes":{"results":[{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20mode'')","type":"DHuS.Attribute"},"Id":"Instrument + mode","Name":"Instrument mode","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Footprint'')","type":"DHuS.Attribute"},"Id":"Footprint","Name":"Footprint","ContentType":"text/plain","ContentLength":"313","Value":" 27.413303,-12.934627 + 28.92095,-12.602928 29.343267,-15.259075 27.838463,-15.552944 27.413303,-12.934627 ","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''JTS%20footprint'')","type":"DHuS.Attribute"},"Id":"JTS + footprint","Name":"JTS footprint","ContentType":"text/plain","ContentLength":"119","Value":"POLYGON + ((-12.934627 27.413303, -12.602928 28.92095, -15.259075 29.343267, -15.552944 + 27.838463, -12.934627 27.413303))","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Orbit%20number%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Orbit + number (start)","Name":"Orbit number (start)","ContentType":"text/plain","ContentLength":"5","Value":"53645","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Relative%20orbit%20%28start%29'')","type":"DHuS.Attribute"},"Id":"Relative + orbit (start)","Name":"Relative orbit (start)","ContentType":"text/plain","ContentLength":"2","Value":"23","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Format'')","type":"DHuS.Attribute"},"Id":"Format","Name":"Format","ContentType":"text/plain","ContentLength":"0","Value":"","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20start'')","type":"DHuS.Attribute"},"Id":"Sensing + start","Name":"Sensing start","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:54:43.105Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Sensing%20stop'')","type":"DHuS.Attribute"},"Id":"Sensing + stop","Name":"Sensing stop","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:55:08.103Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20name'')","type":"DHuS.Attribute"},"Id":"Satellite + name","Name":"Satellite name","ContentType":"text/plain","ContentLength":"10","Value":"SENTINEL-1","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Satellite%20number'')","type":"DHuS.Attribute"},"Id":"Satellite + number","Name":"Satellite number","ContentType":"text/plain","ContentLength":"1","Value":"A","Category":"platform"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Filename'')","type":"DHuS.Attribute"},"Id":"Filename","Name":"Filename","ContentType":"text/plain","ContentLength":"72","Value":"S1A_IW_GRDH_1SDV_20240429T065443_20240429T065508_053645_0683C0_FAB4.SAFE","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument'')","type":"DHuS.Attribute"},"Id":"Instrument","Name":"Instrument","ContentType":"text/plain","ContentLength":"3","Value":"SAR","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Date'')","type":"DHuS.Attribute"},"Id":"Date","Name":"Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T06:54:43.105Z","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Size'')","type":"DHuS.Attribute"},"Id":"Size","Name":"Size","ContentType":"text/plain","ContentLength":"7","Value":"1.64 + GB","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20type'')","type":"DHuS.Attribute"},"Id":"Product + type","Name":"Product type","ContentType":"text/plain","ContentLength":"10","Value":"IW_GRDH_1S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Pass%20direction'')","type":"DHuS.Attribute"},"Id":"Pass + direction","Name":"Pass direction","ContentType":"text/plain","ContentLength":"10","Value":"DESCENDING","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Ingestion%20Date'')","type":"DHuS.Attribute"},"Id":"Ingestion + Date","Name":"Ingestion Date","ContentType":"text/plain","ContentLength":"24","Value":"2024-04-29T07:33:15.552Z","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Identifier'')","type":"DHuS.Attribute"},"Id":"Identifier","Name":"Identifier","ContentType":"text/plain","ContentLength":"67","Value":"S1A_IW_GRDH_1SDV_20240429T065443_20240429T065508_053645_0683C0_FAB4","Category":"summary"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Timeliness%20Category'')","type":"DHuS.Attribute"},"Id":"Timeliness + Category","Name":"Timeliness Category","ContentType":"text/plain","ContentLength":"6","Value":"NRT-3h","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Instrument%20swath'')","type":"DHuS.Attribute"},"Id":"Instrument + swath","Name":"Instrument swath","ContentType":"text/plain","ContentLength":"2","Value":"IW","Category":"instrument"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Mission%20datatake%20id'')","type":"DHuS.Attribute"},"Id":"Mission + datatake id","Name":"Mission datatake id","ContentType":"text/plain","ContentLength":"6","Value":"426944","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20class'')","type":"DHuS.Attribute"},"Id":"Product + class","Name":"Product class","ContentType":"text/plain","ContentLength":"1","Value":"S","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Product%20composition'')","type":"DHuS.Attribute"},"Id":"Product + composition","Name":"Product composition","ContentType":"text/plain","ContentLength":"5","Value":"Slice","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Cycle%20number'')","type":"DHuS.Attribute"},"Id":"Cycle + number","Name":"Cycle number","ContentType":"text/plain","ContentLength":"3","Value":"321","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Polarisation'')","type":"DHuS.Attribute"},"Id":"Polarisation","Name":"Polarisation","ContentType":"text/plain","ContentLength":"5","Value":"VV&VH","Category":"product"},{"__metadata":{"id":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","uri":"https://colhub.met.no/odata/v1/Products/Attributes(''Slice%20number'')","type":"DHuS.Attribute"},"Id":"Slice + number","Name":"Slice number","ContentType":"text/plain","ContentLength":"2","Value":"25","Category":"product"}]},"Class":{"__deferred":{"uri":"https://colhub.met.no/odata/v1/Products(''82aa1c63-6ef7-40e3-ba1a-40fe80e4d808'')/Class"}}}]}}' + content_type: text/plain + method: GET + status: 200 + url: https://colhub.met.no/odata/v1/Products?$format=json&$filter=substringof('IW_GRDH',Name)%20and%20IngestionDate%20gt%20datetime'2024-04-29T07:33:00.000'&$expand=Attributes diff --git a/tests/test_dhus.py b/tests/test_dhus.py new file mode 100644 index 0000000..667dda2 --- /dev/null +++ b/tests/test_dhus.py @@ -0,0 +1,67 @@ +"""Tests for the dhus watcher.""" + + +import datetime as dt + +import responses +import responses._recorder +from freezegun import freeze_time +from pytroll_watchers.dhus_watcher import file_generator, generate_download_links, generate_download_links_since + +server = "https://colhub.met.no" + + +@responses.activate +def test_generate_download_links(): + """Test the generation of download links.""" + responses._add_from_file(file_path="tests/dhus_responses.yaml") + + filter_params = ["substringof('IW_GRDH',Name)", + "IngestionDate gt datetime'2024-04-29T07:33:00.000'"] + + links = list(generate_download_links(server, filter_params)) + assert len(links) == 5 + path, mda = links[0] + assert path.as_uri() == "https://colhub.met.no/odata/v1/Products('e49f8de5-1647-4a7b-ba69-268f9aa77f42')/$value" + assert mda["boundary"]["coordinates"][0] == [-4.545108, 57.399067] + + assert mda["platform_name"] == "Sentinel-1A" + assert mda["sensor"] == "SAR" + assert "ingestion_date" in mda + assert mda["product_type"] == "IW_GRDH_1S" + assert mda["start_time"] == dt.datetime(2024, 4, 29, 6, 46, 23, 104000, tzinfo=dt.timezone.utc) + assert mda["end_time"] == dt.datetime(2024, 4, 29, 6, 46, 48, 102000, tzinfo=dt.timezone.utc) + assert mda["orbit_number"] == 53645 + assert "checksum" in mda + assert "size" in mda + + +@responses.activate +def test_generate_download_links_since(): + """Test limiting the download links to a given point in time.""" + responses._add_from_file(file_path="tests/dhus_responses.yaml") + + filter_params = ["substringof('IW_GRDH',Name)"] + since = dt.datetime(2024, 4, 29, 7, 33, tzinfo=dt.timezone.utc) + links = list(generate_download_links_since(server, filter_params, since)) + assert len(links) == 5 + path, mda = links[1] + assert path.as_uri() == "https://colhub.met.no/odata/v1/Products('162f165a-23c3-4212-869e-4f38b9dade63')/$value" + assert mda["start_time"] == dt.datetime(2024, 4, 29, 6, 52, 38, 105000, tzinfo=dt.timezone.utc) + + +@freeze_time(dt.datetime(2024, 4, 29, 8, 33, tzinfo=dt.timezone.utc)) +@responses.activate +def test_file_generator(): + """Test the file generator.""" + responses._add_from_file(file_path="tests/dhus_responses.yaml") + + filter_params = ["substringof('IW_GRDH',Name)"] + since = dt.timedelta(hours=1) + polling_interval = dict(minutes=0) + links = list(file_generator(server, filter_params, polling_interval, since)) + assert len(links) == 5 + path, mda = links[3] + assert path.as_uri() == "https://colhub.met.no/odata/v1/Products('a6b214dd-fe39-4f50-aa94-f84d8510cedc')/$value" + assert mda["start_time"] == dt.datetime(2024, 4, 29, 6, 54, 18, 105000, tzinfo=dt.timezone.utc) + assert "ingestion_date" not in mda From db7875bb24f7ca9fcfe9ff48e2116f08e85f6b27 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 29 Apr 2024 14:15:12 +0200 Subject: [PATCH 02/14] Use entry points to handle backends --- pyproject.toml | 7 ++++++ src/pytroll_watchers/main_interface.py | 33 ++++++++++---------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3d5e2c0..e9b9122 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,13 @@ license = {file = "LICENSE.txt"} [project.scripts] pytroll-watcher = "pytroll_watchers.main_interface:cli" +[project.entry-points."pytroll_watchers.backends"] +local = "pytroll_watchers.local_watcher" +minio = "pytroll_watchers.minio_notification_watcher" +dataspace = "pytroll_watchers.dataspace_watcher" +datastore = "pytroll_watchers.datastore_watcher" +dhus = "pytroll_watchers.dhus_watcher" + [project.urls] "Documentation" = "https://pytroll-watchers.readthedocs.io/en/latest/" diff --git a/src/pytroll_watchers/main_interface.py b/src/pytroll_watchers/main_interface.py index 2d97c0c..1fe123b 100644 --- a/src/pytroll_watchers/main_interface.py +++ b/src/pytroll_watchers/main_interface.py @@ -2,15 +2,10 @@ import argparse import logging.config +from importlib.metadata import entry_points import yaml -from pytroll_watchers.dhus_watcher import file_publisher as dhus_publisher -from pytroll_watchers.local_watcher import file_generator as local_generator -from pytroll_watchers.local_watcher import file_publisher as local_publisher -from pytroll_watchers.minio_notification_watcher import file_generator as minio_generator -from pytroll_watchers.minio_notification_watcher import file_publisher as minio_publisher - def get_publisher_for_backend(backend): """Get the right publisher for the given backend. @@ -22,14 +17,17 @@ def get_publisher_for_backend(backend): >>> file_publisher(fs_config, publisher_config, message_config) """ - if backend == "minio": - return minio_publisher - elif backend == "local": - return local_publisher - elif backend == "dhus": - return dhus_publisher - else: - raise ValueError(f"Unknown backend {backend}.") + return get_backend(backend).file_publisher + + +def get_backend(backend): + """Get the module for a given backend.""" + eps = entry_points(group="pytroll_watchers.backends") + for ep in eps: + if backend == ep.name: + return ep.load() + raise ValueError(f"Unknown backend {backend}.") + def get_generator_for_backend(backend): """Get the right generator for the given backend. @@ -42,12 +40,7 @@ def get_generator_for_backend(backend): ... # do something with filename and file_metadata """ - if backend == "minio": - return minio_generator - elif backend == "local": - return local_generator - else: - raise ValueError(f"Unknown backend {backend}.") + return get_backend(backend).file_generator def publish_from_config(config): From c84e4a628de300012adabd63f62d845ff15e81c8 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 29 Apr 2024 14:20:08 +0200 Subject: [PATCH 03/14] Fix dependencies --- .github/workflows/ci.yml | 2 +- docs/source/conf.py | 2 +- pyproject.toml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9887fe..88e4e57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: python -m pip install --upgrade pip python -m pip install ruff pytest pytest-cov freezegun responses python -m pip install git+https://github.com/gorakhargosh/watchdog - python -m pip install -e .[local,minio,publishing,ssh,dataspace] + python -m pip install -e .[local,minio,publishing,ssh,dataspace,datastore,dhus] - name: Lint with ruff run: | ruff check . diff --git a/docs/source/conf.py b/docs/source/conf.py index 09d3b0c..bc03029 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ extensions = ["sphinx.ext.napoleon", "sphinx.ext.autodoc"] autodoc_mock_imports = ["watchdog", "minio", "posttroll", "pytest", "trollsift", "universal_path", - "freezegun", "responses", "oauthlib", "requests_oauthlib"] + "freezegun", "responses", "oauthlib", "requests_oauthlib", "defusedxml"] templates_path = ["_templates"] exclude_patterns = [] diff --git a/pyproject.toml b/pyproject.toml index e9b9122..b380fed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ local = ["watchdog"] publishing = ["posttroll"] ssh = ["paramiko"] dataspace = ["oauthlib", "requests_oauthlib", "s3fs"] +datastore = ["oauthlib", "requests_oauthlib"] dhus = ["defusedxml"] [build-system] From 360c4bbfb703700f93fec0ac8c5fe705aaccb946 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 29 Apr 2024 14:27:12 +0200 Subject: [PATCH 04/14] Fix fromisoformat for python 3.10 --- src/pytroll_watchers/dataspace_watcher.py | 8 ++++---- src/pytroll_watchers/dhus_watcher.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pytroll_watchers/dataspace_watcher.py b/src/pytroll_watchers/dataspace_watcher.py index 07871fe..58c29f3 100644 --- a/src/pytroll_watchers/dataspace_watcher.py +++ b/src/pytroll_watchers/dataspace_watcher.py @@ -105,18 +105,18 @@ def run_every(interval): def update_last_publication_date(last_publication_date, metadata): """Update the last publication data based on the metadata.""" - publication_date = _fromisoformat(metadata) + publication_date = _fromisoformat(metadata["PublicationDate"]) if publication_date > last_publication_date: last_publication_date = publication_date return last_publication_date -def _fromisoformat(metadata): +def _fromisoformat(datestring): try: - return datetime.datetime.fromisoformat(metadata["PublicationDate"]) + return datetime.datetime.fromisoformat(datestring) except ValueError: # for python 3.10 - return datetime.datetime.strptime(metadata["PublicationDate"], "%Y-%m-%dT%H:%M:%S.%f%z") + return datetime.datetime.strptime(datestring, "%Y-%m-%dT%H:%M:%S.%f%z") def generate_download_links_since(filter_string, dataspace_auth, last_publication_date, storage_options): diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index c738fb1..c580693 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -13,7 +13,7 @@ from geojson import Polygon from upath import UPath -from pytroll_watchers.dataspace_watcher import run_every +from pytroll_watchers.dataspace_watcher import _fromisoformat, run_every from pytroll_watchers.publisher import file_publisher_from_generator logger = logging.getLogger(__name__) @@ -109,10 +109,10 @@ def generate_download_links(server, filter_params): results_dict = construct_results_dict(entry) mda["platform_name"] = results_dict["Satellite name"].capitalize() + results_dict["Satellite number"] mda["sensor"] = results_dict["Instrument"] - mda["ingestion_date"] = dt.datetime.fromisoformat(results_dict["Ingestion Date"]) + mda["ingestion_date"] = _fromisoformat(results_dict["Ingestion Date"]) mda["product_type"] = results_dict["Product type"] - mda["start_time"] = dt.datetime.fromisoformat(results_dict["Sensing start"]) - mda["end_time"] = dt.datetime.fromisoformat(results_dict["Sensing stop"]) + mda["start_time"] = _fromisoformat(results_dict["Sensing start"]) + mda["end_time"] = _fromisoformat(results_dict["Sensing stop"]) mda["orbit_number"] = int(results_dict["Orbit number (start)"]) mda["checksum"] = dict(algorithm=entry["Checksum"]["Algorithm"], hash=entry["Checksum"]["Value"]) From 002e77883b7ddc0ae59f310bec39954ecb00a74e Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 29 Apr 2024 14:56:42 +0200 Subject: [PATCH 05/14] Fix dataspace metadata --- src/pytroll_watchers/dataspace_watcher.py | 28 ++++++++++++++++++++-- src/pytroll_watchers/dhus_watcher.py | 2 +- tests/test_copernicus_dataspace_watcher.py | 8 ++++++- tests/test_dhus.py | 2 +- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/pytroll_watchers/dataspace_watcher.py b/src/pytroll_watchers/dataspace_watcher.py index 58c29f3..ae6aa8a 100644 --- a/src/pytroll_watchers/dataspace_watcher.py +++ b/src/pytroll_watchers/dataspace_watcher.py @@ -105,7 +105,7 @@ def run_every(interval): def update_last_publication_date(last_publication_date, metadata): """Update the last publication data based on the metadata.""" - publication_date = _fromisoformat(metadata["PublicationDate"]) + publication_date = _fromisoformat(metadata.pop("PublicationDate")) if publication_date > last_publication_date: last_publication_date = publication_date return last_publication_date @@ -149,7 +149,31 @@ def generate_download_links(filter_string, dataspace_auth, storage_options): metadatas = resp.get("value", []) for metadata in metadatas: s3path = UPath("s3://" + metadata["S3Path"], **storage_options) - yield s3path, metadata + mda = dict() + attributes = construct_attributes_dict(metadata) + mda["platform_name"] = attributes["platformShortName"].capitalize() + attributes["platformSerialIdentifier"] + mda["sensor"] = attributes["instrumentShortName"].lower() + mda["PublicationDate"] = metadata["PublicationDate"] + mda["boundary"] = metadata["GeoFootprint"] + mda["product_type"] = attributes["productType"] + mda["start_time"] = _fromisoformat(attributes["beginningDateTime"]) + mda["end_time"] = _fromisoformat(attributes["endingDateTime"]) + mda["orbit_number"] = int(attributes["orbitNumber"]) + + for checksum in metadata["Checksum"]: + if checksum["Algorithm"] == "MD5": + mda["checksum"] = dict(algorithm=checksum["Algorithm"], hash=checksum["Value"]) + break + mda["size"] = int(metadata["ContentLength"]) + + yield s3path, mda + + +def construct_attributes_dict(entry): + """Construct a dict from then "results" item in entry.""" + results = entry["Attributes"] + attributes = {result["Name"]: result["Value"] for result in results} + return attributes @lru_cache(maxsize=1) diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index c580693..6d9518a 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -131,7 +131,7 @@ def extract_boundary(entry): gml, nsmap = read_gml(entry["ContentGeometry"]) boundary_text = gml.find("gml:outerBoundaryIs/gml:LinearRing/gml:coordinates", namespaces=nsmap).text boundary_list = (coords.split(",") for coords in boundary_text.strip().split(" ")) - boundary = Polygon([(float(lon), float(lat)) for (lat, lon) in boundary_list]) + boundary = Polygon([[(float(lon), float(lat)) for (lat, lon) in boundary_list]]) return boundary def read_gml(gml_string): diff --git a/tests/test_copernicus_dataspace_watcher.py b/tests/test_copernicus_dataspace_watcher.py index d89b96f..efc7079 100644 --- a/tests/test_copernicus_dataspace_watcher.py +++ b/tests/test_copernicus_dataspace_watcher.py @@ -24,7 +24,13 @@ def test_dataspace_watcher(): s3path, metadata = files[0] assert s3path.storage_options == storage_options fname = "S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3" - assert metadata["Name"] == fname + assert s3path.as_uri().endswith(fname) + assert metadata["platform_name"] == "Sentinel-3B" + assert metadata["sensor"] == "olci" + assert metadata["start_time"] == datetime.datetime(2024, 4, 15, 7, 40, 29, 480000, tzinfo=datetime.timezone.utc) + assert metadata["end_time"] == datetime.datetime(2024, 4, 15, 7, 43, 29, 480000, tzinfo=datetime.timezone.utc) + assert metadata["boundary"]["coordinates"][0][0] == [67.8172, 69.3862] + assert metadata["orbit_number"] == 31107 @freeze_time(datetime.datetime.now(datetime.timezone.utc)) diff --git a/tests/test_dhus.py b/tests/test_dhus.py index 667dda2..83315c0 100644 --- a/tests/test_dhus.py +++ b/tests/test_dhus.py @@ -23,7 +23,7 @@ def test_generate_download_links(): assert len(links) == 5 path, mda = links[0] assert path.as_uri() == "https://colhub.met.no/odata/v1/Products('e49f8de5-1647-4a7b-ba69-268f9aa77f42')/$value" - assert mda["boundary"]["coordinates"][0] == [-4.545108, 57.399067] + assert mda["boundary"]["coordinates"][0][0] == [-4.545108, 57.399067] assert mda["platform_name"] == "Sentinel-1A" assert mda["sensor"] == "SAR" From 02d355eb2907fa9bb796ee0a4cda47e13ecd01e6 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 29 Apr 2024 15:03:41 +0200 Subject: [PATCH 06/14] Privatize some functions --- src/pytroll_watchers/dataspace_watcher.py | 8 +++---- src/pytroll_watchers/dhus_watcher.py | 28 +++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/pytroll_watchers/dataspace_watcher.py b/src/pytroll_watchers/dataspace_watcher.py index ae6aa8a..3b3014e 100644 --- a/src/pytroll_watchers/dataspace_watcher.py +++ b/src/pytroll_watchers/dataspace_watcher.py @@ -77,7 +77,7 @@ def file_generator(filter_string, for next_check in run_every(polling_interval): generator = generate_download_links_since(filter_string, dataspace_auth, last_pub_date, storage_options) for s3path, metadata in generator: - last_pub_date = update_last_publication_date(last_pub_date, metadata) + last_pub_date = _update_last_publication_date(last_pub_date, metadata) yield s3path, metadata logger.info("Finished polling.") if next_check > datetime.datetime.now(datetime.timezone.utc): @@ -103,7 +103,7 @@ def run_every(interval): break -def update_last_publication_date(last_publication_date, metadata): +def _update_last_publication_date(last_publication_date, metadata): """Update the last publication data based on the metadata.""" publication_date = _fromisoformat(metadata.pop("PublicationDate")) if publication_date > last_publication_date: @@ -150,7 +150,7 @@ def generate_download_links(filter_string, dataspace_auth, storage_options): for metadata in metadatas: s3path = UPath("s3://" + metadata["S3Path"], **storage_options) mda = dict() - attributes = construct_attributes_dict(metadata) + attributes = _construct_attributes_dict(metadata) mda["platform_name"] = attributes["platformShortName"].capitalize() + attributes["platformSerialIdentifier"] mda["sensor"] = attributes["instrumentShortName"].lower() mda["PublicationDate"] = metadata["PublicationDate"] @@ -169,7 +169,7 @@ def generate_download_links(filter_string, dataspace_auth, storage_options): yield s3path, mda -def construct_attributes_dict(entry): +def _construct_attributes_dict(entry): """Construct a dict from then "results" item in entry.""" results = entry["Attributes"] attributes = {result["Name"]: result["Value"] for result in results} diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index 6d9518a..0bf0342 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -64,14 +64,14 @@ def file_generator(server, filter_params, polling_interval, start_from=None): for next_check in run_every(polling_interval): generator = generate_download_links_since(server, filter_params, last_pub_date) for path, metadata in generator: - last_pub_date = update_last_publication_date(last_pub_date, metadata) + last_pub_date = _update_last_publication_date(last_pub_date, metadata) yield path, metadata logger.info("Finished polling.") if next_check > dt.datetime.now(dt.timezone.utc): logger.info(f"Next iteration at {next_check}") -def update_last_publication_date(last_publication_date, metadata): +def _update_last_publication_date(last_publication_date, metadata): """Update the last publication data based on the metadata.""" publication_date = metadata.pop("ingestion_date") if publication_date > last_publication_date: @@ -105,28 +105,28 @@ def generate_download_links(server, filter_params): for entry in entries["d"]["results"]: mda = dict() path = UPath(entry["__metadata"]["media_src"]) - mda["boundary"] = extract_boundary(entry) - results_dict = construct_results_dict(entry) - mda["platform_name"] = results_dict["Satellite name"].capitalize() + results_dict["Satellite number"] - mda["sensor"] = results_dict["Instrument"] - mda["ingestion_date"] = _fromisoformat(results_dict["Ingestion Date"]) - mda["product_type"] = results_dict["Product type"] - mda["start_time"] = _fromisoformat(results_dict["Sensing start"]) - mda["end_time"] = _fromisoformat(results_dict["Sensing stop"]) - mda["orbit_number"] = int(results_dict["Orbit number (start)"]) + mda["boundary"] = _extract_boundary_as_geojson(entry) + attributes = _construct_attributes_dict(entry) + mda["platform_name"] = attributes["Satellite name"].capitalize() + attributes["Satellite number"] + mda["sensor"] = attributes["Instrument"] + mda["ingestion_date"] = _fromisoformat(attributes["Ingestion Date"]) + mda["product_type"] = attributes["Product type"] + mda["start_time"] = _fromisoformat(attributes["Sensing start"]) + mda["end_time"] = _fromisoformat(attributes["Sensing stop"]) + mda["orbit_number"] = int(attributes["Orbit number (start)"]) mda["checksum"] = dict(algorithm=entry["Checksum"]["Algorithm"], hash=entry["Checksum"]["Value"]) mda["size"] = int(entry["ContentLength"]) yield path, mda -def construct_results_dict(entry): +def _construct_attributes_dict(entry): """Construct a dict from then "results" item in entry.""" results = entry["Attributes"]["results"] - results_dict = {result["Id"]: result["Value"] for result in results} + results_dict = {result["Name"]: result["Value"] for result in results} return results_dict -def extract_boundary(entry): +def _extract_boundary_as_geojson(entry): """Extract the boundary from the entry metadata.""" gml, nsmap = read_gml(entry["ContentGeometry"]) boundary_text = gml.find("gml:outerBoundaryIs/gml:LinearRing/gml:coordinates", namespaces=nsmap).text From 38c820c50576d5f6a526c4fad75b04239d677c54 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 30 Apr 2024 11:55:20 +0200 Subject: [PATCH 07/14] Fix datastor mda handling --- src/pytroll_watchers/datastore_watcher.py | 14 ++++++++++++-- tests/test_datastore.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/pytroll_watchers/datastore_watcher.py b/src/pytroll_watchers/datastore_watcher.py index 7854cd4..4b25c65 100644 --- a/src/pytroll_watchers/datastore_watcher.py +++ b/src/pytroll_watchers/datastore_watcher.py @@ -92,8 +92,18 @@ def generate_download_links(search_params, ds_auth): path = UPath(links[0]["href"], encoded=True, client_kwargs=client_args) # In the future, it might be interesting to generate items from the sip-entries, as # they contain individual files for zip archives. - - yield path, feature + mda = dict() + mda["boundary"] = feature["geometry"] + acq_info = feature["properties"]["acquisitionInformation"][0] + mda["platform_name"] = acq_info["platform"]["platformShortName"] + mda["sensor"] = acq_info["instrument"]["instrumentShortName"].lower() + mda["orbit_number"] = acq_info["acquisitionParameters"]["orbitNumber"] + start_string, end_string = feature["properties"]["date"].split("/") + mda["start_time"] = datetime.datetime.fromisoformat(start_string) + mda["end_time"] = datetime.datetime.fromisoformat(end_string) + mda["product_type"] = str(collection) + mda["checksum"] = dict(algorithm="md5", hash=feature["properties"]["extraInformation"]["md5"]) + yield path, mda class DatastoreOAuth2Session(): diff --git a/tests/test_datastore.py b/tests/test_datastore.py index 3cb62ea..d1744a8 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -48,7 +48,14 @@ def test_datastore_get_download_links_since(search_params): path, mda = features[0] assert str(path) == "https://api.eumetsat.int/data/download/1.0.0/collections/EO%3AEUM%3ADAT%3A0407/products/S3B_OL_2_WFR____20240416T104217_20240416T104517_20240417T182315_0180_092_051_1980_MAR_O_NT_003.SEN3" assert expected_token in path.storage_options["client_kwargs"]["headers"]["Authorization"] - assert "sip-entries" in mda["properties"]["links"] + assert mda["boundary"]["coordinates"][0][0] == [-14.3786, 52.4516] + assert mda["platform_name"] == "Sentinel-3B" + assert mda["sensor"] == "olci" + assert mda["start_time"] == datetime.datetime(2024, 4, 16, 10, 42, 16, 954262, tzinfo=datetime.timezone.utc) + assert mda["end_time"] == datetime.datetime(2024, 4, 16, 10, 45, 16, 954262, tzinfo=datetime.timezone.utc) + assert mda["orbit_number"] == 31123 + assert mda["product_type"] == "EO:EUM:DAT:0407" + assert mda["checksum"] == dict(algorithm="md5", hash="9057eb08f2a4e9f4c5a8d2eeaacedaef") @contextmanager @@ -98,7 +105,6 @@ def test_datastore_file_generator(tmp_path, search_params): path, mda = features[0] assert str(path) == "https://api.eumetsat.int/data/download/1.0.0/collections/EO%3AEUM%3ADAT%3A0407/products/S3B_OL_2_WFR____20240416T104217_20240416T104517_20240417T182315_0180_092_051_1980_MAR_O_NT_003.SEN3" assert expected_token in path.storage_options["client_kwargs"]["headers"]["Authorization"] - assert "sip-entries" in mda["properties"]["links"] @freeze_time(datetime.datetime.now(datetime.timezone.utc)) From 2fc72b60a0e5fb6379d4130f01af3e4817007d4d Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 30 Apr 2024 12:02:59 +0200 Subject: [PATCH 08/14] Restructure doc --- docs/source/backends.rst | 27 ++++++++ docs/source/cli.rst | 21 ++++++ docs/source/index.rst | 134 ++------------------------------------ docs/source/other_api.rst | 12 ++++ docs/source/published.rst | 65 ++++++++++++++++++ tests/test_datastore.py | 2 +- 6 files changed, 131 insertions(+), 130 deletions(-) create mode 100644 docs/source/backends.rst create mode 100644 docs/source/cli.rst create mode 100644 docs/source/other_api.rst create mode 100644 docs/source/published.rst diff --git a/docs/source/backends.rst b/docs/source/backends.rst new file mode 100644 index 0000000..551b316 --- /dev/null +++ b/docs/source/backends.rst @@ -0,0 +1,27 @@ +Available backends +================== + +Local watcher +------------- +.. automodule:: pytroll_watchers.local_watcher + :members: + +Minio bucket notification watcher +--------------------------------- +.. automodule:: pytroll_watchers.minio_notification_watcher + :members: + +Copernicus dataspace watcher +---------------------------- +.. automodule:: pytroll_watchers.dataspace_watcher + :members: + +EUMETSAT datastore watcher +-------------------------- +.. automodule:: pytroll_watchers.datastore_watcher + :members: + +DHuS watcher +------------ +.. automodule:: pytroll_watchers.dhus_watcher + :members: diff --git a/docs/source/cli.rst b/docs/source/cli.rst new file mode 100644 index 0000000..885c4e4 --- /dev/null +++ b/docs/source/cli.rst @@ -0,0 +1,21 @@ +CLI +*** + +The command-line tool can be used by invoking `pytroll-watcher `. An example config-file can be:: + + backend: minio + fs_config: + endpoint_url: my_endpoint.pytroll.org + bucket_name: satellite-data-viirs + storage_options: + profile: profile_for_credentials + publisher_config: + name: viirs_watcher + message_config: + subject: /segment/viirs/l1b/ + atype: file + data: + sensor: viirs + aliases: + platform_name: + npp: Suomi-NPP diff --git a/docs/source/index.rst b/docs/source/index.rst index 95b2596..0795ad0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,137 +10,13 @@ Welcome to pytroll-watchers's documentation! :maxdepth: 2 :caption: Contents: -Pytroll-watcher is a library and command-line tool to detect changes on a local or remote file system. - -At the moment we support local filesystems and Minio S3 buckets through bucket notifications. - -CLI -*** - -The command-line tool can be used by invoking `pytroll-watcher `. An example config-file can be:: - - backend: minio - fs_config: - endpoint_url: my_endpoint.pytroll.org - bucket_name: satellite-data-viirs - storage_options: - profile: profile_for_credentials - publisher_config: - name: viirs_watcher - message_config: - subject: /segment/viirs/l1b/ - atype: file - data: - sensor: viirs - aliases: - platform_name: - npp: Suomi-NPP - -Published messages -****************** - -The published messages will contain information on how to access the resource advertized. The following parameters will -be present in the message. - -uid ---- - -This is the unique identifier for the resource. In general, it is the basename for the file/objects, since we assume -that two files with the same name will have the same content. In some cases it can include the containing directory. - -Examples of uids: - - - `SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` - - `S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` - -uri ---- - -This is the URI that can be used to access the resource. The URI can be composed as fsspec allows for more complex cases. - -Examples of uris: - - - `s3://viirs-data/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` - - `zip://sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5::s3://viirs-data/viirs_sdr_npp_d20240408_t1006227_e1007469_b64498.zip` - - `https://someplace.com/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` - - filesystem - ---------- - - Sometimes the URI is not enough to gain access to the resource, for example when the hosting service requires - authentification. This is why pytroll-watchers with also provide the filesystem and the path items. The filesystem - parameter is the fsspec json representation of the filesystem. This can be used on the recipient side using eg:: - - fsspec.AbstractFileSystem.from_json(json.dumps(fs_info)) - -where `fs_info` is the content of the filesystem parameter. - -To pass authentification parameters to the filesystem, use the `storage_options` configuration item. - - -Example of filesystem: - - - `{"cls": "s3fs.core.S3FileSystem", "protocol": "s3", "args": [], "profile": "someprofile"}` - -.. warning:: - - Pytroll-watchers tries to prevent publishing of sensitive information such as passwords and secret keys, and will - raise an error in most cases when this is done. However, always double-check your pytroll-watchers configuration so - that secrets are not passed to the library to start with. - Solutions include ssh-agent for ssh-based filesystems, storing credentials in .aws config files for s3 filesystems. - For http-based filesystems implemented in pytroll-watchers, the username and password are used to generate a token - prior to publishing, and will thus not be published. - -path ----- - -This parameter is the companion to `filesystem` and gives the path to the resource within the filesystem. - -Examples of paths: - - - `/viirs-data/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` - - `/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` - - `/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` - - -API -*** - -Main interface --------------- -.. automodule:: pytroll_watchers - :members: - -Local watcher -------------- -.. automodule:: pytroll_watchers.local_watcher - :members: - -Minio bucket notification watcher ---------------------------------- -.. automodule:: pytroll_watchers.minio_notification_watcher - :members: - -Copernicus dataspace watcher ----------------------------- -.. automodule:: pytroll_watchers.dataspace_watcher - :members: - -EUMETSAT datastore watcher --------------------------- -.. automodule:: pytroll_watchers.datastore_watcher - :members: - -DHuS watcher ------------- -.. automodule:: pytroll_watchers.dhus_watcher - :members: + cli + published + backends + other_api +Pytroll-watcher is a library and command-line tool to detect changes on a local or remote file system. -Testing utilities ------------------ -.. automodule:: pytroll_watchers.testing - :members: Indices and tables ================== diff --git a/docs/source/other_api.rst b/docs/source/other_api.rst new file mode 100644 index 0000000..b8d522e --- /dev/null +++ b/docs/source/other_api.rst @@ -0,0 +1,12 @@ +Common API +********** + +Main interface +-------------- +.. automodule:: pytroll_watchers + :members: + +Testing utilities +----------------- +.. automodule:: pytroll_watchers.testing + :members: diff --git a/docs/source/published.rst b/docs/source/published.rst new file mode 100644 index 0000000..d9371d2 --- /dev/null +++ b/docs/source/published.rst @@ -0,0 +1,65 @@ +Published messages +****************** + +The published messages will contain information on how to access the resource advertized. The following parameters will +be present in the message. + +uid +--- + +This is the unique identifier for the resource. In general, it is the basename for the file/objects, since we assume +that two files with the same name will have the same content. In some cases it can include the containing directory. + +Examples of uids: + + - `SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` + - `S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` + +uri +--- + +This is the URI that can be used to access the resource. The URI can be composed as fsspec allows for more complex cases. + +Examples of uris: + + - `s3://viirs-data/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` + - `zip://sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5::s3://viirs-data/viirs_sdr_npp_d20240408_t1006227_e1007469_b64498.zip` + - `https://someplace.com/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` + + filesystem + ---------- + + Sometimes the URI is not enough to gain access to the resource, for example when the hosting service requires + authentification. This is why pytroll-watchers with also provide the filesystem and the path items. The filesystem + parameter is the fsspec json representation of the filesystem. This can be used on the recipient side using eg:: + + fsspec.AbstractFileSystem.from_json(json.dumps(fs_info)) + +where `fs_info` is the content of the filesystem parameter. + +To pass authentification parameters to the filesystem, use the `storage_options` configuration item. + + +Example of filesystem: + + - `{"cls": "s3fs.core.S3FileSystem", "protocol": "s3", "args": [], "profile": "someprofile"}` + +.. warning:: + + Pytroll-watchers tries to prevent publishing of sensitive information such as passwords and secret keys, and will + raise an error in most cases when this is done. However, always double-check your pytroll-watchers configuration so + that secrets are not passed to the library to start with. + Solutions include ssh-agent for ssh-based filesystems, storing credentials in .aws config files for s3 filesystems. + For http-based filesystems implemented in pytroll-watchers, the username and password are used to generate a token + prior to publishing, and will thus not be published. + +path +---- + +This parameter is the companion to `filesystem` and gives the path to the resource within the filesystem. + +Examples of paths: + + - `/viirs-data/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` + - `/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` + - `/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` diff --git a/tests/test_datastore.py b/tests/test_datastore.py index d1744a8..64d193a 100644 --- a/tests/test_datastore.py +++ b/tests/test_datastore.py @@ -102,7 +102,7 @@ def test_datastore_file_generator(tmp_path, search_params): expected_token = "eceba4e1-95e6-3526-8c42-c3c9dc14ff5c" # noqa assert len(features) == 5 - path, mda = features[0] + path, _ = features[0] assert str(path) == "https://api.eumetsat.int/data/download/1.0.0/collections/EO%3AEUM%3ADAT%3A0407/products/S3B_OL_2_WFR____20240416T104217_20240416T104517_20240417T182315_0180_092_051_1980_MAR_O_NT_003.SEN3" assert expected_token in path.storage_options["client_kwargs"]["headers"]["Authorization"] From e5d53bda16a9c2b26593bd033981e4fd307fb02a Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 30 Apr 2024 12:07:19 +0200 Subject: [PATCH 09/14] Add to documentation --- docs/source/published.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/source/published.rst b/docs/source/published.rst index d9371d2..947f000 100644 --- a/docs/source/published.rst +++ b/docs/source/published.rst @@ -4,6 +4,9 @@ Published messages The published messages will contain information on how to access the resource advertized. The following parameters will be present in the message. +Resource location information +============================= + uid --- @@ -26,6 +29,7 @@ Examples of uris: - `zip://sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5::s3://viirs-data/viirs_sdr_npp_d20240408_t1006227_e1007469_b64498.zip` - `https://someplace.com/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` + filesystem ---------- @@ -63,3 +67,17 @@ Examples of paths: - `/viirs-data/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` - `/sdr/SVM13_npp_d20240408_t1006227_e1007469_b64498_c20240408102334392250_cspp_dev.h5` - `/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` + +Other metadata +============== + +Other metadata items are provided when possible: + +* boundary: the geojson boundary of the data +* platform_name +* sensor +* orbit_number +* start_time +* end_time +* product_type +* checksum From 881528203ca52865f2ce12f091f6b9170511a39b Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 30 Apr 2024 12:12:38 +0200 Subject: [PATCH 10/14] Fix isoformat parsing --- src/pytroll_watchers/common.py | 32 +++++++++++++++++++++ src/pytroll_watchers/dataspace_watcher.py | 34 +++-------------------- src/pytroll_watchers/datastore_watcher.py | 6 ++-- src/pytroll_watchers/dhus_watcher.py | 8 +++--- 4 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 src/pytroll_watchers/common.py diff --git a/src/pytroll_watchers/common.py b/src/pytroll_watchers/common.py new file mode 100644 index 0000000..18ce272 --- /dev/null +++ b/src/pytroll_watchers/common.py @@ -0,0 +1,32 @@ +"""Collection of function needed by multiple watchers.""" + +import datetime +import time + + +def run_every(interval): + """Generator that ticks every `interval`. + + Args: + interval: the timedelta object giving the amount of time to wait between ticks. An interval of 0 will just make + tick once, then return (and thus busy loops aren't allowed). + + Yields: + The time of the next tick. + """ + while True: + next_check = datetime.datetime.now(datetime.timezone.utc) + interval + yield next_check + to_wait = max(next_check.timestamp() - time.time(), 0) + time.sleep(to_wait) + if not interval: # interval is 0 + break + + +def fromisoformat(datestring): + """Wrapper around datetime's fromisoformat that also works on python 3.10.""" + try: + return datetime.datetime.fromisoformat(datestring) + except ValueError: + # for python 3.10 + return datetime.datetime.strptime(datestring, "%Y-%m-%dT%H:%M:%S.%f%z") diff --git a/src/pytroll_watchers/dataspace_watcher.py b/src/pytroll_watchers/dataspace_watcher.py index 3b3014e..7f1d272 100644 --- a/src/pytroll_watchers/dataspace_watcher.py +++ b/src/pytroll_watchers/dataspace_watcher.py @@ -19,6 +19,7 @@ from requests_oauthlib import OAuth2Session from upath import UPath +from pytroll_watchers.common import fromisoformat, run_every from pytroll_watchers.publisher import file_publisher_from_generator client_id = "cdse-public" @@ -84,41 +85,14 @@ def file_generator(filter_string, logger.info(f"Next iteration at {next_check}") -def run_every(interval): - """Generator that ticks every `interval`. - - Args: - interval: the timedelta object giving the amount of time to wait between ticks. An interval of 0 will just make - tick once, then return (and thus busy loops aren't allowed). - - Yields: - The time of the next tick. - """ - while True: - next_check = datetime.datetime.now(datetime.timezone.utc) + interval - yield next_check - to_wait = max(next_check.timestamp() - time.time(), 0) - time.sleep(to_wait) - if not interval: # interval is 0 - break - - def _update_last_publication_date(last_publication_date, metadata): """Update the last publication data based on the metadata.""" - publication_date = _fromisoformat(metadata.pop("PublicationDate")) + publication_date = fromisoformat(metadata.pop("PublicationDate")) if publication_date > last_publication_date: last_publication_date = publication_date return last_publication_date -def _fromisoformat(datestring): - try: - return datetime.datetime.fromisoformat(datestring) - except ValueError: - # for python 3.10 - return datetime.datetime.strptime(datestring, "%Y-%m-%dT%H:%M:%S.%f%z") - - def generate_download_links_since(filter_string, dataspace_auth, last_publication_date, storage_options): """Generate download links for data that was published since a given `last publication_date`. @@ -156,8 +130,8 @@ def generate_download_links(filter_string, dataspace_auth, storage_options): mda["PublicationDate"] = metadata["PublicationDate"] mda["boundary"] = metadata["GeoFootprint"] mda["product_type"] = attributes["productType"] - mda["start_time"] = _fromisoformat(attributes["beginningDateTime"]) - mda["end_time"] = _fromisoformat(attributes["endingDateTime"]) + mda["start_time"] = fromisoformat(attributes["beginningDateTime"]) + mda["end_time"] = fromisoformat(attributes["endingDateTime"]) mda["orbit_number"] = int(attributes["orbitNumber"]) for checksum in metadata["Checksum"]: diff --git a/src/pytroll_watchers/datastore_watcher.py b/src/pytroll_watchers/datastore_watcher.py index 4b25c65..d565c22 100644 --- a/src/pytroll_watchers/datastore_watcher.py +++ b/src/pytroll_watchers/datastore_watcher.py @@ -18,7 +18,7 @@ from requests_oauthlib import OAuth2Session from upath import UPath -from pytroll_watchers.dataspace_watcher import run_every +from pytroll_watchers.common import fromisoformat, run_every from pytroll_watchers.publisher import file_publisher_from_generator logger = logging.getLogger(__name__) @@ -99,8 +99,8 @@ def generate_download_links(search_params, ds_auth): mda["sensor"] = acq_info["instrument"]["instrumentShortName"].lower() mda["orbit_number"] = acq_info["acquisitionParameters"]["orbitNumber"] start_string, end_string = feature["properties"]["date"].split("/") - mda["start_time"] = datetime.datetime.fromisoformat(start_string) - mda["end_time"] = datetime.datetime.fromisoformat(end_string) + mda["start_time"] = fromisoformat(start_string) + mda["end_time"] = fromisoformat(end_string) mda["product_type"] = str(collection) mda["checksum"] = dict(algorithm="md5", hash=feature["properties"]["extraInformation"]["md5"]) yield path, mda diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index 0bf0342..c6e9973 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -13,7 +13,7 @@ from geojson import Polygon from upath import UPath -from pytroll_watchers.dataspace_watcher import _fromisoformat, run_every +from pytroll_watchers.common import fromisoformat, run_every from pytroll_watchers.publisher import file_publisher_from_generator logger = logging.getLogger(__name__) @@ -109,10 +109,10 @@ def generate_download_links(server, filter_params): attributes = _construct_attributes_dict(entry) mda["platform_name"] = attributes["Satellite name"].capitalize() + attributes["Satellite number"] mda["sensor"] = attributes["Instrument"] - mda["ingestion_date"] = _fromisoformat(attributes["Ingestion Date"]) + mda["ingestion_date"] = fromisoformat(attributes["Ingestion Date"]) mda["product_type"] = attributes["Product type"] - mda["start_time"] = _fromisoformat(attributes["Sensing start"]) - mda["end_time"] = _fromisoformat(attributes["Sensing stop"]) + mda["start_time"] = fromisoformat(attributes["Sensing start"]) + mda["end_time"] = fromisoformat(attributes["Sensing stop"]) mda["orbit_number"] = int(attributes["Orbit number (start)"]) mda["checksum"] = dict(algorithm=entry["Checksum"]["Algorithm"], hash=entry["Checksum"]["Value"]) From fa45478f1255dc06a2ab592a9fa5aa963945880d Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Tue, 30 Apr 2024 12:17:34 +0200 Subject: [PATCH 11/14] Fix indentation --- docs/source/published.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/published.rst b/docs/source/published.rst index 947f000..2908fbd 100644 --- a/docs/source/published.rst +++ b/docs/source/published.rst @@ -30,8 +30,8 @@ Examples of uris: - `https://someplace.com/files/S3B_OL_1_EFR____20240415T074029_20240415T074329_20240415T094236_0179_092_035_1620_PS2_O_NR_003.SEN3/Oa02_radiances.nc` - filesystem - ---------- +filesystem +---------- Sometimes the URI is not enough to gain access to the resource, for example when the hosting service requires authentification. This is why pytroll-watchers with also provide the filesystem and the path items. The filesystem From 5d5dc9b0df6964a6642e8b63def49f4caea55605 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 2 May 2024 08:58:00 +0200 Subject: [PATCH 12/14] Test publishing --- src/pytroll_watchers/dhus_watcher.py | 1 + tests/test_dhus.py | 37 +++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index c6e9973..f21057d 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -117,6 +117,7 @@ def generate_download_links(server, filter_params): mda["checksum"] = dict(algorithm=entry["Checksum"]["Algorithm"], hash=entry["Checksum"]["Value"]) mda["size"] = int(entry["ContentLength"]) + mda["uid"] = attributes["Filename"] yield path, mda def _construct_attributes_dict(entry): diff --git a/tests/test_dhus.py b/tests/test_dhus.py index 83315c0..4dae53b 100644 --- a/tests/test_dhus.py +++ b/tests/test_dhus.py @@ -6,7 +6,14 @@ import responses import responses._recorder from freezegun import freeze_time -from pytroll_watchers.dhus_watcher import file_generator, generate_download_links, generate_download_links_since +from posttroll.message import Message +from posttroll.testing import patched_publisher +from pytroll_watchers.dhus_watcher import ( + file_generator, + file_publisher, + generate_download_links, + generate_download_links_since, +) server = "https://colhub.met.no" @@ -65,3 +72,31 @@ def test_file_generator(): assert path.as_uri() == "https://colhub.met.no/odata/v1/Products('a6b214dd-fe39-4f50-aa94-f84d8510cedc')/$value" assert mda["start_time"] == dt.datetime(2024, 4, 29, 6, 54, 18, 105000, tzinfo=dt.timezone.utc) assert "ingestion_date" not in mda + assert mda["uid"] == "S1A_IW_GRDH_1SDV_20240429T065418_20240429T065443_053645_0683C0_8D1D.SAFE" + + +@freeze_time(dt.datetime(2024, 4, 29, 8, 33, tzinfo=dt.timezone.utc)) +@responses.activate +def test_publish_paths(): + """Test publishing paths.""" + responses._add_from_file(file_path="tests/dhus_responses.yaml") + + filter_params = ["substringof('IW_GRDH',Name)"] + + publisher_settings = dict(nameservers=False, port=1979) + message_settings = dict(subject="/segment/olci/l1b/", atype="file", aliases=dict(sensor={"SAR": "SAR-C"})) + + with patched_publisher() as messages: + fs_config = dict(server=server, + filter_params=filter_params, + polling_interval=dict(minutes=0), + start_from=dict(hours=1)) + + + file_publisher(fs_config=fs_config, + publisher_config=publisher_settings, + message_config=message_settings) + message = Message(rawstr=messages[3]) + + assert message.data["uid"] == "S1A_IW_GRDH_1SDV_20240429T065418_20240429T065443_053645_0683C0_8D1D.SAFE" + assert message.data["sensor"] == "SAR-C" From 1dd8e9795e6d36f6586fcf4ff7b6579c6ca97a9e Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 2 May 2024 09:21:50 +0200 Subject: [PATCH 13/14] Add example to dhus doc --- src/pytroll_watchers/dhus_watcher.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index f21057d..5fcf108 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -2,6 +2,31 @@ For more information about DHuS, check out https://sentineldatahub.github.io/DataHubSystem/about.html + + +An example configuration file to retrieve Sentinel 1 data from a DHuS instance:: + + .. code-block:: yaml + + backend: dhus + fs_config: + server: https://myhub.someplace.org/ + filter_params: + - substringof('IW_GRDH',Name) + polling_interval: + seconds: 10 + start_from: + hours: 6 + publisher_config: + name: s1_watcher + message_config: + subject: /segment/s1/l1b/ + atype: file + aliases: + sensor: + SAR: SAR-C + + """ import datetime as dt From 3f7d5e35d3080ce3f079c0f226673781b3fa1654 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 2 May 2024 09:30:12 +0200 Subject: [PATCH 14/14] Fix doc formatting --- src/pytroll_watchers/dhus_watcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pytroll_watchers/dhus_watcher.py b/src/pytroll_watchers/dhus_watcher.py index 5fcf108..a8f26ba 100644 --- a/src/pytroll_watchers/dhus_watcher.py +++ b/src/pytroll_watchers/dhus_watcher.py @@ -4,9 +4,9 @@ https://sentineldatahub.github.io/DataHubSystem/about.html -An example configuration file to retrieve Sentinel 1 data from a DHuS instance:: +An example configuration file to retrieve Sentinel 1 data from a DHuS instance: - .. code-block:: yaml +.. code-block:: yaml backend: dhus fs_config: