Skip to content

Commit

Permalink
Merge pull request #25 from mraspaud/fix-orbit-number-for-geo
Browse files Browse the repository at this point in the history 
Introduce a geo example for datastore and test
  • Loading branch information
@mraspaud
mraspaud authored Aug 22, 2024
2 parents ed5b6b2 + c30cb96 commit 7e66bb2
Show file tree
Hide file tree
Showing 4 changed files with 200 additions and 6 deletions.
22 changes: 19 additions & 3 deletions src/pytroll_watchers/datastore_watcher.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,21 @@
Note:
The links produced can only be downloaded with a valid token. A token comes with the links, but
has only a limited validity time.
has only a limited validity time (maybe 5 minutes).
An example for getting links to MSG data::
from pytroll_watchers.datastore_watcher import generate_download_links_since
ds_auth = dict(netrc_host="api.eumetsat.int")
collection = "EO:EUM:DAT:MSG:HRSEVIRI"
search_params = dict(collection=collection)
now = datetime.datetime.now(datetime.timezone.utc)
yesterday = now - datetime.timedelta(hours=6)
links = list(generate_download_links_since(search_params, ds_auth, yesterday))
"""

Expand All @@ -20,6 +34,7 @@

from pytroll_watchers.common import fromisoformat, run_every
from pytroll_watchers.publisher import file_publisher_from_generator
from pytroll_watchers.version import version

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -97,7 +112,8 @@ def generate_download_links(search_params, ds_auth):
acq_info = feature["properties"]["acquisitionInformation"][0]
mda["platform_name"] = acq_info["platform"]["platformShortName"]
mda["sensor"] = acq_info["instrument"]["instrumentShortName"].lower()
mda["orbit_number"] = acq_info["acquisitionParameters"]["orbitNumber"]
with suppress(KeyError):
mda["orbit_number"] = acq_info["acquisitionParameters"]["orbitNumber"]
start_string, end_string = feature["properties"]["date"].split("/")
mda["start_time"] = fromisoformat(start_string)
mda["end_time"] = fromisoformat(end_string)
Expand Down Expand Up @@ -125,7 +141,7 @@ def get(self, params):
self.fetch_token()
search_url = f"{data_url}/search-products/1.0.0/os"
headers = {"referer": "https://github.com/pytroll/pytroll-watchers",
"User-Agent": "pytroll-watchers / 0.1.0"}
"User-Agent": f"pytroll-watchers / {version}"}

return self._oauth.get(search_url, params=params, headers=headers).json()

Expand Down
8 changes: 6 additions & 2 deletions src/pytroll_watchers/publisher.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,14 @@ def file_publisher_from_generator(generator, publisher_config, message_config):
def unpack(path, archive_format):
"""Unpack the path and yield the extracted filenames."""
import fsspec
fs = fsspec.get_filesystem_class(archive_format)(fsspec.open(path))
fs = fsspec.get_filesystem_class(archive_format)(fsspec.open(path.path, **path.storage_options))
files = fs.find("/")
for fi in files:
yield UPath(fi, protocol=archive_format, target_protocol=path.protocol, fo=path.as_uri())
yield UPath(fi,
protocol=archive_format,
target_protocol=path.protocol,
target_options=path.storage_options,
fo=path.as_uri())


def _build_file_location(file_item):
Expand Down
132 changes: 132 additions & 0 deletions tests/datastore_responses_geo.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
responses:
- response:
auto_calculate_content_length: false
body: '{"access_token":"62f55741-7241-3d0d-ba18-0ac541e42ac3","scope":"am_application_scope
default","token_type":"Bearer","expires_in":187}'
content_type: text/plain
headers:
access-control-allow-headers: authorization,Access-Control-Allow-Origin,Content-Type,SOAPAction
access-control-allow-methods: POST
access-control-allow-origin: ''
access-control-expose-headers: ''
cache-control: no-store
#content-type: application/json
date: Mon, 19 Aug 2024 12:58:59 GMT
pragma: no-cache
strict-transport-security: max-age=15724800; includeSubDomains
transfer-encoding: chunked
x-content-type-options: nosniff
x-frame-options: DENY
x-xss-protection: 1; mode=block
method: POST
status: 200
url: https://api.eumetsat.int/token
- response:
auto_calculate_content_length: false
body: "\n { \"id\" : \"https:\\/\\/api.eumetsat.int\\/data\\/search-products\\\
/1.0.0\\/os?pi=EO:EUM:DAT:MSG:HRSEVIRI&publication=%5B2023-11-11T08%3A00%3A00.000%2C2023-11-11T08%3A15%3A00.000%5D&si=0&c=100&\"\
,\n \"type\" : \"FeatureCollection\",\n \"totalResults\" : 1,\n \"\
itemsPerPage\" : 100,\n \"startIndex\" : 0,\n \"queries\" : \n { \"\
request\" : \n [ \n { \"count\" : 100,\n \"startIndex\"\
\ : 0,\n \"eo:parentIdentifier\" : \"EO:EUM:DAT:MSG:HRSEVIRI\",\n \
\ \"eo:publicationDate\" : \"[2023-11-11T08:00:00.000,2023-11-11T08:15:00.000]\"\
\ } ] },\n \"properties\" : \n { \"type\" : \"Properties\",\n \"\
title\" : \"EUMETSAT EOP OpenSearch (EOPOS) results feed for EO:EUM:DAT:MSG:HRSEVIRI\"\
,\n \"subTitle\" : \"Search results for your query pi=EO:EUM:DAT:MSG:HRSEVIRI&publication=%5B2023-11-11T08%3A00%3A00.000%2C2023-11-11T08%3A15%3A00.000%5D&si=0&c=100&\"\
,\n \"rights\" : \"Copyright\",\n \"updated\" : \"2024-08-19T12:58:59.802Z\"\
,\n \"subject\" : \"climatologyMeteorologyAtmosphere\",\n \"creator\"\
\ : \"EUMETSAT\",\n \"lang\" : \"en\",\n \"parameters\" : \n \
\ [ \n { \"name\" : \"sat\",\n \"value\" : \"platformShortName\"\
,\n \"title\" : \"Mission \\/ Satellite\",\n \"minimum\" :\
\ 0,\n \"options\" : \n [ \n { \"value\" : \"MSG3\"\
,\n \"results\" : 1 } ] },\n \n { \"name\" : \"type\"\
,\n \"value\" : \"productType\",\n \"title\" : \"Product Type\"\
,\n \"minimum\" : 0,\n \"options\" : \n [ \n \
\ { \"value\" : \"MSG15\",\n \"results\" : 1 } ] },\n \
\ \n { \"name\" : \"dtstart\",\n \"value\" : \"start\"\
,\n \"title\" : \"Temporal Start\",\n \"minimum\" : 0,\n \
\ \"minInclusive\" : \"2023-11-11T08:00:09.969Z\",\n \"maxInclusive\"\
\ : \"2023-11-11T08:00:09.969Z\" },\n \n { \"name\" : \"dtend\"\
,\n \"value\" : \"end\",\n \"title\" : \"Temporal End\",\n\
\ \"minimum\" : 0,\n \"minInclusive\" : \"2023-11-11T08:12:42.335Z\"\
,\n \"maxInclusive\" : \"2023-11-11T08:12:42.335Z\" },\n \n\
\ { \"name\" : \"publication\",\n \"value\" : \"publicationDate\"\
,\n \"title\" : \"publication date\",\n \"minimum\" : 0,\n\
\ \"minInclusive\" : \"2023-11-11T08:13:42.277Z\",\n \"maxInclusive\"\
\ : \"2023-11-11T08:13:42.277Z\" } ],\n \"links\" : \n { \"type\"\
\ : \"Links\",\n \"search\" : \n { \"type\" : \"application\\\
/opensearchdescription+xml\",\n \"href\" : \"https:\\/\\/api.eumetsat.int\\\
/data\\/search-products\\/1.0.0\\/osdd?pi=EO:EUM:DAT:MSG:HRSEVIRI\",\n \
\ \"title\" : \"OpenSearch Description Document\" },\n \"self\" :\
\ \n { \"type\" : \"application\\/json\",\n \"href\" : \"https:\\\
/\\/api.eumetsat.int\\/data\\/search-products\\/1.0.0\\/os?pi=EO:EUM:DAT:MSG:HRSEVIRI&publication=%5B2023-11-11T08%3A00%3A00.000%2C2023-11-11T08%3A15%3A00.000%5D&si=0&c=100&format=json\"\
,\n \"title\" : \"this search\" },\n \"first\" : \n {\
\ \"type\" : \"application\\/json\",\n \"href\" : \"https:\\/\\/api.eumetsat.int\\\
/data\\/search-products\\/1.0.0\\/os?pi=EO:EUM:DAT:MSG:HRSEVIRI&publication=%5B2023-11-11T08%3A00%3A00.000%2C2023-11-11T08%3A15%3A00.000%5D&c=100&format=json\"\
,\n \"title\" : \"first result page\" },\n \"last\" : \n \
\ { \"type\" : \"application\\/json\",\n \"href\" : \"https:\\\
/\\/api.eumetsat.int\\/data\\/search-products\\/1.0.0\\/os?pi=EO:EUM:DAT:MSG:HRSEVIRI&publication=%5B2023-11-11T08%3A00%3A00.000%2C2023-11-11T08%3A15%3A00.000%5D&si=0&c=100&format=json\"\
,\n \"title\" : \"last result page\" } },\n \"profiles\" : \n\
\ [ \n { \"href\" : \"http:\\/\\/www.opengis.net\\/spec\\/owc-geojson\\\
/1.0\\/req\\/core\" },\n \n { \"href\" : \"http:\\/\\/www.opengis.net\\\
/spec\\/os-geojson\\/1.0\\/req\\/core\" } ] },\n \"features\" : \n [ \n\
\ { \"type\" : \"Feature\",\n \"id\" : \"MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\"\
,\n \"geometry\" : \n { },\n \"properties\" : \n \
\ { \"type\" : \"Properties\",\n \"identifier\" : \"MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\"\
,\n \"parentIdentifier\" : \"EO:EUM:DAT:MSG:HRSEVIRI\",\n \
\ \"title\" : \"MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\",\n \
\ \"date\" : \"2023-11-11T08:00:09.969Z\\/2023-11-11T08:12:42.335Z\"\
,\n \"updated\" : \"2023-11-11T08:13:42.277Z\",\n \"acquisitionInformation\"\
\ : \n [ \n { \"platform\" : \n { \"platformShortName\"\
\ : \"MSG3\" },\n \"instrument\" : \n { \"instrumentShortName\"\
\ : \"SEVIRI\" },\n \"acquisitionParameters\" : \n \
\ { } } ],\n \"productInformation\" : \n { \"productType\"\
\ : \"MSG15\",\n \"size\" : 174135 },\n \"extraInformation\"\
\ : \n { \"md5\" : \"b33bb9a82cdf18676fcc0db1d1e8a728\" },\n \
\ \"links\" : \n { \"type\" : \"Links\",\n \"data\" :\
\ \n [ \n { \"type\" : \"Link\",\n \"\
href\" : \"https:\\/\\/api.eumetsat.int\\/data\\/download\\/1.0.0\\/collections\\\
/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI\\/products\\/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\"\
,\n \"mediaType\" : \"application\\/zip\",\n \"\
title\" : \"Product download\" } ],\n \"alternates\" : \n \
\ [ \n { \"type\" : \"Link\",\n \"href\" : \"\
https:\\/\\/api.eumetsat.int\\/data\\/download\\/1.0.0\\/collections\\/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI\\\
/products\\/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\\/metadata\"\
,\n \"mediaType\" : \"application\\/xml\",\n \"\
title\" : \"Metadata\" },\n \n { \"type\" : \"Link\"\
,\n \"href\" : \"https:\\/\\/api.eumetsat.int\\/data\\/download\\\
/1.0.0\\/collections\\/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI\\/products\\/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\\\
/metadata?format=json\",\n \"mediaType\" : \"application\\/json\"\
,\n \"title\" : \"Metadata in JSON format\" } ],\n \
\ \"sip-entries\" : \n [ \n { \"type\" : \"Link\",\n\
\ \"href\" : \"https:\\/\\/api.eumetsat.int\\/data\\/download\\\
/1.0.0\\/collections\\/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI\\/products\\/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\\\
/entry?name=MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA.nat\",\n \
\ \"mediaType\" : \"application\\/octet-stream\",\n \
\ \"title\" : \"MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA.nat\"\
\ },\n \n { \"type\" : \"Link\",\n \
\ \"href\" : \"https:\\/\\/api.eumetsat.int\\/data\\/download\\/1.0.0\\/collections\\\
/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI\\/products\\/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\\\
/entry?name=EOPMetadata.xml\",\n \"mediaType\" : \"application\\\
/xml\",\n \"title\" : \"EOPMetadata.xml\" },\n \n\
\ { \"type\" : \"Link\",\n \"href\" : \"https:\\\
/\\/api.eumetsat.int\\/data\\/download\\/1.0.0\\/collections\\/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI\\\
/products\\/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA\\/entry?name=manifest.xml\"\
,\n \"mediaType\" : \"application\\/xml\",\n \"\
title\" : \"manifest.xml\" } ] } } } ] }"
content_type: text/plain
headers:
access-control-allow-headers: Origin, Accept, X-Requested-With, Content-Type,
Access-Control-Request-Method, Access-Control-Request-Headers
access-control-allow-methods: GET, HEAD, POST, PUT, DELETE, TRACE, OPTIONS,
CONNECT, PATCH
access-control-allow-origin: '*'
access-control-expose-headers: ''
access-control-max-age: '3600'
#content-type: application/json
date: Mon, 19 Aug 2024 12:58:59 GMT
strict-transport-security: max-age=15724800; includeSubDomains
transfer-encoding: chunked
method: GET
status: 200
url: https://api.eumetsat.int/data/search-products/1.0.0/os?format=json&pi=EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI&si=0&c=100&publication=%5B2023-11-11T08%3A00%3A00.000%2C2023-11-11T08%3A15%3A00.000%5D
44 changes: 43 additions & 1 deletion tests/test_datastore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@
from freezegun import freeze_time
from posttroll.message import Message
from posttroll.testing import patched_publisher
from pytroll_watchers.datastore_watcher import file_generator, file_publisher, generate_download_links_since
from pytroll_watchers.datastore_watcher import (
file_generator,
file_publisher,
generate_download_links,
generate_download_links_since,
)


@pytest.fixture()
Expand Down Expand Up @@ -107,6 +112,43 @@ def test_datastore_file_generator(tmp_path, search_params):
assert expected_token in path.storage_options["client_kwargs"]["headers"]["Authorization"]


@pytest.fixture()
def search_params_geo():
"""Generate the search parameters for the tests."""
collection = "EO:EUM:DAT:MSG:HRSEVIRI"
publication_start = datetime.datetime(2023, 11, 11, 8, 0)
publication_end = datetime.datetime(2023, 11, 11, 8, 15)
str_pub_start = publication_start.isoformat(timespec="milliseconds")
str_pub_end = publication_end.isoformat(timespec="milliseconds")
return dict(collection=collection, publication=f"[{str_pub_start},{str_pub_end}]")


@responses.activate
def test_datastore_generate_download_links_geo(tmp_path, search_params_geo):
"""Test the file generator."""
netrc_host = "myitem"
netrc_file = tmp_path / "netrc"

with open(netrc_file, "w") as fd:
fd.write(f"machine {netrc_host} login [email protected] password mypassword")

ds_auth = dict(netrc_host=netrc_host, netrc_file=netrc_file)

response_file = "tests/datastore_responses_geo.yaml"
responses._add_from_file(file_path=response_file)

features = list(generate_download_links(search_params_geo, ds_auth))

assert len(features) == 1

path, mda = features[0]
assert "orbit_number" not in mda
assert str(path) == "https://api.eumetsat.int/data/download/1.0.0/collections/EO%3AEUM%3ADAT%3AMSG%3AHRSEVIRI/products/MSG3-SEVI-MSG15-0100-NA-20231111081242.335000000Z-NA"
assert mda["start_time"] == datetime.datetime(2023, 11, 11, 8, 0, 9, 969000, tzinfo=datetime.timezone.utc)
assert mda["end_time"] == datetime.datetime(2023, 11, 11, 8, 12, 42, 335000, tzinfo=datetime.timezone.utc)



@freeze_time(datetime.datetime.now(datetime.timezone.utc))
def test_publish_paths(caplog, search_params):
"""Test publishing paths."""
Expand Down

0 comments on commit 7e66bb2

Please sign in to comment.