Skip to content

Commit

Permalink
Change backoff strategy for tenacity, and return value from last atte…
Browse files Browse the repository at this point in the history
…mpt even if it failed.
  • Loading branch information
rwiker committed Nov 7, 2023
1 parent a5f1611 commit 6be74bc
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 14 deletions.
10 changes: 6 additions & 4 deletions src/sumo/wrapper/_blob_client.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import httpx

from ._decorators import is_retryable_exception, is_retryable_status_code, raise_for_status, raise_for_status_async
from tenacity import retry, retry_if_exception, retry_if_result, wait_exponential_jitter, stop_after_attempt
from ._decorators import is_retryable_exception, is_retryable_status_code, raise_for_status, raise_for_status_async, return_last_value
from tenacity import retry, retry_if_exception, retry_if_result, wait_exponential, wait_random_exponential, stop_after_attempt

class BlobClient:
"""Upload blobs to blob store using pre-authorized URLs"""
Expand All @@ -12,8 +12,9 @@ class BlobClient:
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
def upload_blob(self, blob: bytes, url: str):
"""Upload a blob.
Expand All @@ -38,8 +39,9 @@ def upload_blob(self, blob: bytes, url: str):
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
async def upload_blob_async(self, blob: bytes, url: str):
"""Upload a blob async.
Expand Down
4 changes: 4 additions & 0 deletions src/sumo/wrapper/_decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ def is_retryable_status_code(response):
return response.status_code in [502, 503, 504]


def return_last_value(retry_state):
return retry_state.outcome.result()


def http_retry(func):
return tn.retry(
func,
Expand Down
28 changes: 18 additions & 10 deletions src/sumo/wrapper/sumo_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@
from .config import APP_REGISTRATION, TENANT_ID, AUTHORITY_HOST_URI

from ._decorators import is_retryable_exception, \
is_retryable_status_code, raise_for_status, raise_for_status_async
is_retryable_status_code, raise_for_status, raise_for_status_async, return_last_value
from tenacity import retry, retry_if_exception, retry_if_result, \
wait_exponential_jitter, stop_after_attempt
wait_exponential, wait_random_exponential, stop_after_attempt

logger = logging.getLogger("sumo.wrapper")

Expand Down Expand Up @@ -117,8 +117,9 @@ def blob_client(self) -> BlobClient:
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
def get(self, path: str, params: dict = None) -> dict:
"""Performs a GET-request to the Sumo API.
Expand Down Expand Up @@ -171,8 +172,9 @@ def get(self, path: str, params: dict = None) -> dict:
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
def post(
self,
Expand Down Expand Up @@ -251,8 +253,9 @@ def post(
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
def put(
self, path: str, blob: bytes = None, json: dict = None
Expand Down Expand Up @@ -303,8 +306,9 @@ def put(
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
def delete(self, path: str, params: dict = None) -> dict:
"""Performs a DELETE-request to the Sumo API.
Expand Down Expand Up @@ -365,8 +369,9 @@ def getLogger(self, name):
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
async def get_async(self, path: str, params: dict = None):
"""Performs an async GET-request to the Sumo API.
Expand Down Expand Up @@ -418,8 +423,9 @@ async def get_async(self, path: str, params: dict = None):
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
async def post_async(
self,
Expand Down Expand Up @@ -501,8 +507,9 @@ async def post_async(
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
async def put_async(
self, path: str, blob: bytes = None, json: dict = None
Expand Down Expand Up @@ -554,8 +561,9 @@ async def put_async(
retry_if_exception(is_retryable_exception)
| retry_if_result(is_retryable_status_code)
),
wait=wait_exponential_jitter(),
wait=wait_exponential(multiplier=0.5)+wait_random_exponential(multiplier=0.5),
reraise=True,
retry_error_callback=return_last_value,
)
async def delete_async(self, path: str, params: dict = None) -> dict:
"""Performs an async DELETE-request to the Sumo API.
Expand Down

0 comments on commit 6be74bc

Please sign in to comment.