From fe6fefb996586b02cc28e0ab97f49fb2d9fa0974 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Mon, 22 Apr 2024 13:50:05 +0200 Subject: [PATCH] fix: add back-off on requests retry to handle HTTP 429 status --- terracatalogueclient/__init__.py | 2 +- terracatalogueclient/client.py | 14 +++++++++++--- tests/test_cgls.py | 10 ++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/terracatalogueclient/__init__.py b/terracatalogueclient/__init__.py index 61aee1b..6386daf 100644 --- a/terracatalogueclient/__init__.py +++ b/terracatalogueclient/__init__.py @@ -1,5 +1,5 @@ __title__ = "terracatalogueclient" -__version__ = "0.1.16" +__version__ = "0.1.17" __author__ = "Stijn Caerts" from terracatalogueclient.client import Catalogue, Collection, Product, ProductFile, ProductFileType diff --git a/terracatalogueclient/client.py b/terracatalogueclient/client.py index 92afe9b..15840bf 100644 --- a/terracatalogueclient/client.py +++ b/terracatalogueclient/client.py @@ -173,19 +173,27 @@ def __init__(self, config: CatalogueConfig = None): self._auth = None self.s3 = None - adapter = requests.adapters.HTTPAdapter(max_retries=requests.adapters.Retry(total=5)) + adapter = requests.adapters.HTTPAdapter( + max_retries=requests.adapters.Retry( + total=5, + backoff_factor=2, + status_forcelist=[429, 500, 502, 503, 504] + ) + ) self._session_search = requests.Session() self._session_search.headers.update(_DEFAULT_REQUEST_HEADERS) self._session_search.headers.update({ "Accept": "application/json, application/geo+json" }) - self._session_search.mount("http", adapter) + self._session_search.mount("http://", adapter) + self._session_search.mount("https://", adapter) self._session_download = requests.Session() self._session_download.headers.update(_DEFAULT_REQUEST_HEADERS) self._session_download.headers.update({"Accept": "application/json"}) - self._session_download.mount("http", adapter) + self._session_download.mount("http://", adapter) + self._session_download.mount("https://", adapter) def authenticate(self) -> 'Catalogue': """ diff --git a/tests/test_cgls.py b/tests/test_cgls.py index 5e152a9..e15419e 100644 --- a/tests/test_cgls.py +++ b/tests/test_cgls.py @@ -48,3 +48,13 @@ def test_download(self): with tempfile.TemporaryDirectory() as tmpdir: catalogue.download_product(next(products), tmpdir) + def test_download_multiple(self): + catalogue = Catalogue(self.config_cgls) + params = { + "collection": "clms_global_lst_5km_v2_hourly_netcdf", + "start": date(2023, 1, 1), + "end": date(2023, 1, 2) + } + products = catalogue.get_products(**params) + with tempfile.TemporaryDirectory() as tmpdir: + catalogue.download_products(products, tmpdir, force=True)