From c0a6e5ce62e984a7a1745f7b455ea4060fa0342b Mon Sep 17 00:00:00 2001 From: tbascoul Date: Thu, 1 Sep 2022 15:34:11 +0200 Subject: [PATCH] Make asset requirements for remote storage optional (#167) * assets: make s3 optional, add error when it is not installed * assets: make gcs optional, add error when it is not installed * assets: make azure optional, add error when it is not installed * lint: lint remote.py * update docs with optional dependencies * remove tf pinned version on optional requirements * update requirements files move optional dependencies to requirements-optional * update cli and asset drivers for optional imports Co-authored-by: Victor Benichoux Co-authored-by: Cyril Le Mat <7548748+CyrilLeMat@users.noreply.github.com> --- README.md | 2 + docs/assets/storage_provider.md | 13 ++-- modelkit/assets/cli.py | 32 +++++++- modelkit/assets/drivers/azure.py | 14 ++-- modelkit/assets/drivers/gcs.py | 16 ++-- modelkit/assets/drivers/retry.py | 37 ++++----- modelkit/assets/drivers/s3.py | 14 ++-- modelkit/assets/remote.py | 39 +++++++++- requirements-dev.in | 1 + requirements-dev.txt | 124 ++++++++++--------------------- requirements-optional.txt | 10 ++- requirements.in | 4 +- requirements.txt | 100 +++---------------------- setup.cfg | 10 ++- tests/assets/test_retry.py | 24 ++++-- 15 files changed, 203 insertions(+), 237 deletions(-) diff --git a/README.md b/README.md index 9c589050..53485988 100644 --- a/README.md +++ b/README.md @@ -70,5 +70,7 @@ Install with `pip`: pip install modelkit ``` +Optional dependencies are available for remote storage providers ([see documentation](https://cornerstone-ondemand.github.io/modelkit/assets/storage_provider/#using-different-providers)) + ## Community Join our [community](https://discord.gg/ayj5wdAArV) on Discord to get support and leave feedback diff --git a/docs/assets/storage_provider.md b/docs/assets/storage_provider.md index fb3314ee..59f3967e 100644 --- a/docs/assets/storage_provider.md +++ b/docs/assets/storage_provider.md @@ -28,11 +28,14 @@ Developers may additionally need to be able to push new assets and or update exi ## Using different providers -The flavor of the remote store that is used depends on the `MODELKIT_STORAGE_PROVIDER` environment variables +The flavor of the remote store that is used depends on optional dependencies used during pip install and on the `MODELKIT_STORAGE_PROVIDER` environment variable. + +The default `pip install modelkit` will only allow you to target a local directory. + ### Using AWS S3 storage -Use `MODELKIT_STORAGE_PROVIDER=s3` to connect to S3 storage. +Use `pip install modelkit[assets-s3]` and setup this environment variable `MODELKIT_STORAGE_PROVIDER=s3` to connect to S3 storage. We use [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) under the hood. @@ -53,7 +56,7 @@ Use `AWS_KMS_KEY_ID` environment variable to set your key and be able to upload ### GCS storage -Use `MODELKIT_STORAGE_PROVIDER=gcs` to connect to GCS storage. +Use `pip install modelkit[assets-gcs]` and setup this environment variable `MODELKIT_STORAGE_PROVIDER=gcs` to connect to GCS storage. We use [google-cloud-storage](https://googleapis.dev/python/storage/latest/index.html). @@ -67,7 +70,7 @@ If `GOOGLE_APPLICATION_CREDENTIALS` is provided, it should point to a local JSON ### Using Azure blob storage -Use `MODELKIT_STORAGE_PROVIDER=az` to connect to Azure blob storage. +Use `pip install modelkit[assets-az]` and setup this environment variable `MODELKIT_STORAGE_PROVIDER=az` to connect to Azure blob storage. We use [azure-storage-blobl](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-quickstart-blobs-python) under the hood. @@ -80,7 +83,7 @@ The client is created by passing the authentication information to `BlobServiceC ### `local` mode -Use `MODELKIT_STORAGE_PROVIDER=local` to treat a local folder as a remote source. +Setup this environment variable `MODELKIT_STORAGE_PROVIDER=local` to treat a local folder as a remote source. Assets will be downloaded from this folder to the configured asset dir. diff --git a/modelkit/assets/cli.py b/modelkit/assets/cli.py index 3d4e868f..35d4db96 100755 --- a/modelkit/assets/cli.py +++ b/modelkit/assets/cli.py @@ -10,11 +10,21 @@ from rich.table import Table from rich.tree import Tree -from modelkit.assets.drivers.gcs import GCSStorageDriver -from modelkit.assets.drivers.s3 import S3StorageDriver +try: + from modelkit.assets.drivers.gcs import GCSStorageDriver + + has_gcs = True +except ModuleNotFoundError: + has_gcs = False +try: + from modelkit.assets.drivers.s3 import S3StorageDriver + + has_s3 = True +except ModuleNotFoundError: + has_s3 = False from modelkit.assets.errors import ObjectDoesNotExistError from modelkit.assets.manager import AssetsManager -from modelkit.assets.remote import StorageProvider +from modelkit.assets.remote import DriverNotInstalledError, StorageProvider from modelkit.assets.settings import AssetSpec @@ -121,8 +131,16 @@ def new_(asset_path, asset_spec, storage_prefix, dry_run): if not os.path.exists(asset_path): parsed_path = parse_remote_url(asset_path) if parsed_path["storage_prefix"] == "gs": + if not has_gcs: + raise DriverNotInstalledError( + "GCS driver not installed, install modelkit[assets-gcs]" + ) driver = GCSStorageDriver(bucket=parsed_path["bucket_name"]) elif parsed_path["storage_prefix"] == "s3": + if not has_s3: + raise DriverNotInstalledError( + "S3 driver not installed, install modelkit[assets-s3]" + ) driver = S3StorageDriver(bucket=parsed_path["bucket_name"]) else: raise ValueError( @@ -212,8 +230,16 @@ def update_(asset_path, asset_spec, storage_prefix, bump_major, dry_run): if not os.path.exists(asset_path): parsed_path = parse_remote_url(asset_path) if parsed_path["storage_prefix"] == "gs": + if not has_gcs: + raise DriverNotInstalledError( + "GCS driver not installed, install modelkit[assets-gcs]" + ) driver = GCSStorageDriver(bucket=parsed_path["bucket_name"]) elif parsed_path["storage_prefix"] == "s3": + if not has_s3: + raise DriverNotInstalledError( + "S3 driver not installed, install modelkit[assets-s3]" + ) driver = S3StorageDriver(bucket=parsed_path["bucket_name"]) else: raise ValueError( diff --git a/modelkit/assets/drivers/azure.py b/modelkit/assets/drivers/azure.py index 6e0a9910..5806a82a 100644 --- a/modelkit/assets/drivers/azure.py +++ b/modelkit/assets/drivers/azure.py @@ -7,10 +7,12 @@ from modelkit.assets import errors from modelkit.assets.drivers.abc import StorageDriver -from modelkit.assets.drivers.retry import RETRY_POLICY +from modelkit.assets.drivers.retry import retry_policy logger = get_logger(__name__) +AZURE_RETRY_POLICY = retry_policy() + class AzureStorageDriver(StorageDriver): bucket: str @@ -34,13 +36,13 @@ def __init__( os.environ["AZURE_STORAGE_CONNECTION_STRING"] ) - @retry(**RETRY_POLICY) + @retry(**AZURE_RETRY_POLICY) def iterate_objects(self, prefix=None): container = self.client.get_container_client(self.bucket) for blob in container.list_blobs(prefix=prefix): yield blob["name"] - @retry(**RETRY_POLICY) + @retry(**AZURE_RETRY_POLICY) def upload_object(self, file_path, object_name): blob_client = self.client.get_blob_client( container=self.bucket, blob=object_name @@ -50,7 +52,7 @@ def upload_object(self, file_path, object_name): with open(file_path, "rb") as f: blob_client.upload_blob(f) - @retry(**RETRY_POLICY) + @retry(**AZURE_RETRY_POLICY) def download_object(self, object_name, destination_path): blob_client = self.client.get_blob_client( container=self.bucket, blob=object_name @@ -67,14 +69,14 @@ def download_object(self, object_name, destination_path): with open(destination_path, "wb") as f: f.write(blob_client.download_blob().readall()) - @retry(**RETRY_POLICY) + @retry(**AZURE_RETRY_POLICY) def delete_object(self, object_name): blob_client = self.client.get_blob_client( container=self.bucket, blob=object_name ) blob_client.delete_blob() - @retry(**RETRY_POLICY) + @retry(**AZURE_RETRY_POLICY) def exists(self, object_name): blob_client = self.client.get_blob_client( container=self.bucket, blob=object_name diff --git a/modelkit/assets/drivers/gcs.py b/modelkit/assets/drivers/gcs.py index 991964c3..bdf53c70 100644 --- a/modelkit/assets/drivers/gcs.py +++ b/modelkit/assets/drivers/gcs.py @@ -1,7 +1,7 @@ import os from typing import Optional -from google.api_core.exceptions import NotFound +from google.api_core.exceptions import GoogleAPIError, NotFound from google.cloud import storage from google.cloud.storage import Client from structlog import get_logger @@ -9,10 +9,12 @@ from modelkit.assets import errors from modelkit.assets.drivers.abc import StorageDriver -from modelkit.assets.drivers.retry import RETRY_POLICY +from modelkit.assets.drivers.retry import retry_policy logger = get_logger(__name__) +GCS_RETRY_POLICY = retry_policy(GoogleAPIError) + class GCSStorageDriver(StorageDriver): bucket: str @@ -35,13 +37,13 @@ def __init__( else: self.client = Client() - @retry(**RETRY_POLICY) + @retry(**GCS_RETRY_POLICY) def iterate_objects(self, prefix=None): bucket = self.client.bucket(self.bucket) for blob in bucket.list_blobs(prefix=prefix): yield blob.name - @retry(**RETRY_POLICY) + @retry(**GCS_RETRY_POLICY) def upload_object(self, file_path, object_name): bucket = self.client.bucket(self.bucket) blob = bucket.blob(object_name) @@ -50,7 +52,7 @@ def upload_object(self, file_path, object_name): with open(file_path, "rb") as f: blob.upload_from_file(f) - @retry(**RETRY_POLICY) + @retry(**GCS_RETRY_POLICY) def download_object(self, object_name, destination_path): bucket = self.client.bucket(self.bucket) blob = bucket.blob(object_name) @@ -66,13 +68,13 @@ def download_object(self, object_name, destination_path): driver=self, bucket=self.bucket, object_name=object_name ) - @retry(**RETRY_POLICY) + @retry(**GCS_RETRY_POLICY) def delete_object(self, object_name): bucket = self.client.bucket(self.bucket) blob = bucket.blob(object_name) blob.delete() - @retry(**RETRY_POLICY) + @retry(**GCS_RETRY_POLICY) def exists(self, object_name): bucket = self.client.bucket(self.bucket) blob = bucket.blob(object_name) diff --git a/modelkit/assets/drivers/retry.py b/modelkit/assets/drivers/retry.py index f366c9f1..e9d3baac 100644 --- a/modelkit/assets/drivers/retry.py +++ b/modelkit/assets/drivers/retry.py @@ -1,5 +1,3 @@ -import botocore -import google import requests from structlog import get_logger from tenacity import retry_if_exception, stop_after_attempt, wait_random_exponential @@ -7,14 +5,6 @@ logger = get_logger(__name__) -def retriable_error(exception): - return ( - isinstance(exception, botocore.exceptions.ClientError) - or isinstance(exception, google.api_core.exceptions.GoogleAPIError) - or isinstance(exception, requests.exceptions.ChunkedEncodingError) - ) - - def log_after_retry(retry_state): logger.info( "Retrying", @@ -24,10 +14,23 @@ def log_after_retry(retry_state): ) -RETRY_POLICY = { - "wait": wait_random_exponential(multiplier=1, min=4, max=10), - "stop": stop_after_attempt(5), - "retry": retry_if_exception(retriable_error), - "after": log_after_retry, - "reraise": True, -} +def retry_policy(type_error=None): + if not type_error: + + def is_retry_eligible(error): + return isinstance(error, requests.exceptions.ChunkedEncodingError) + + else: + + def is_retry_eligible(error): + return isinstance(error, type_error) or isinstance( + error, requests.exceptions.ChunkedEncodingError + ) + + return { + "wait": wait_random_exponential(multiplier=1, min=4, max=10), + "stop": stop_after_attempt(5), + "retry": retry_if_exception(is_retry_eligible), + "after": log_after_retry, + "reraise": True, + } diff --git a/modelkit/assets/drivers/s3.py b/modelkit/assets/drivers/s3.py index b58f9d4f..d957e21f 100644 --- a/modelkit/assets/drivers/s3.py +++ b/modelkit/assets/drivers/s3.py @@ -8,10 +8,12 @@ from modelkit.assets import errors from modelkit.assets.drivers.abc import StorageDriver -from modelkit.assets.drivers.retry import RETRY_POLICY +from modelkit.assets.drivers.retry import retry_policy logger = get_logger(__name__) +S3_RETRY_POLICY = retry_policy(botocore.exceptions.ClientError) + class S3StorageDriver(StorageDriver): bucket: str @@ -44,7 +46,7 @@ def __init__( region_name=aws_default_region or os.environ.get("AWS_DEFAULT_REGION"), ) - @retry(**RETRY_POLICY) + @retry(**S3_RETRY_POLICY) def iterate_objects(self, prefix=None): paginator = self.client.get_paginator("list_objects_v2") pages = paginator.paginate(Bucket=self.bucket, Prefix=prefix or "") @@ -52,7 +54,7 @@ def iterate_objects(self, prefix=None): for obj in page.get("Contents", []): yield obj["Key"] - @retry(**RETRY_POLICY) + @retry(**S3_RETRY_POLICY) def upload_object(self, file_path, object_name): if self.aws_kms_key_id: self.client.upload_file( # pragma: no cover @@ -67,7 +69,7 @@ def upload_object(self, file_path, object_name): else: self.client.upload_file(file_path, self.bucket, object_name) - @retry(**RETRY_POLICY) + @retry(**S3_RETRY_POLICY) def download_object(self, object_name, destination_path): try: with open(destination_path, "wb") as f: @@ -81,11 +83,11 @@ def download_object(self, object_name, destination_path): driver=self, bucket=self.bucket, object_name=object_name ) - @retry(**RETRY_POLICY) + @retry(**S3_RETRY_POLICY) def delete_object(self, object_name): self.client.delete_object(Bucket=self.bucket, Key=object_name) - @retry(**RETRY_POLICY) + @retry(**S3_RETRY_POLICY) def exists(self, object_name): try: self.client.head_object(Bucket=self.bucket, Key=object_name) diff --git a/modelkit/assets/remote.py b/modelkit/assets/remote.py index 336bd6c9..1794ebca 100644 --- a/modelkit/assets/remote.py +++ b/modelkit/assets/remote.py @@ -12,10 +12,27 @@ from modelkit.assets import errors from modelkit.assets.drivers.abc import StorageDriver -from modelkit.assets.drivers.azure import AzureStorageDriver -from modelkit.assets.drivers.gcs import GCSStorageDriver + +try: + from modelkit.assets.drivers.azure import AzureStorageDriver + + has_az = True +except ModuleNotFoundError: + has_az = False +try: + from modelkit.assets.drivers.gcs import GCSStorageDriver + + has_gcs = True +except ModuleNotFoundError: + has_gcs = False from modelkit.assets.drivers.local import LocalStorageDriver -from modelkit.assets.drivers.s3 import S3StorageDriver + +try: + from modelkit.assets.drivers.s3 import S3StorageDriver + + has_s3 = True +except ModuleNotFoundError: + has_s3 = False from modelkit.assets.settings import AssetSpec from modelkit.utils.logging import ContextualizedLogging @@ -36,6 +53,10 @@ class UnknownDriverError(Exception): pass +class DriverNotInstalledError(Exception): + pass + + class NoConfiguredProviderError(Exception): pass @@ -69,12 +90,24 @@ def __init__( raise NoConfiguredProviderError() if provider == "gcs": + if not has_gcs: + raise DriverNotInstalledError( + "GCS driver not installed, install modelkit[assets-gcs]" + ) self.driver = GCSStorageDriver(**driver_settings) elif provider == "s3": + if not has_s3: + raise DriverNotInstalledError( + "S3 driver not installed, install modelkit[assets-s3]" + ) self.driver = S3StorageDriver(**driver_settings) elif provider == "local": self.driver = LocalStorageDriver(**driver_settings) elif provider == "az": + if not has_az: + raise DriverNotInstalledError( + "Azure driver not installed, install modelkit[assets-az]" + ) self.driver = AzureStorageDriver(**driver_settings) else: raise UnknownDriverError() diff --git a/requirements-dev.in b/requirements-dev.in index e3cedfd1..0d97b6d4 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -10,6 +10,7 @@ black flake8<4 # flake8==4.0.1 pins importlib-metadata<4.3 which leads to a conflict isort nox +boto3 google-api-core google-cloud-storage azure-storage-blob diff --git a/requirements-dev.txt b/requirements-dev.txt index 4ab617c0..01a760ca 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,10 +2,8 @@ # This file is autogenerated by pip-compile with python 3.10 # To update, run: # -# pip-compile --index-url=https://pypi.python.org/simple --output-file=requirements-dev.txt requirements-dev.in +# pip-compile --no-emit-index-url --output-file=requirements-dev.txt requirements-dev.in # ---index-url https://pypi.python.org/simple - aiohttp==3.8.1 # via # -c requirements.txt @@ -33,26 +31,20 @@ attrs==21.4.0 # -c requirements.txt # aiohttp # pytest -azure-core==1.23.1 +azure-core==1.25.0 # via - # -c requirements.txt # azure-storage-blob -azure-storage-blob==12.11.0 - # via - # -c requirements.txt - # -r requirements-dev.in - # -r requirements.in + # msrest +azure-storage-blob==12.13.1 + # via -r requirements-dev.in backports-entry-points-selectable==1.1.1 # via virtualenv black==22.3.0 # via -r requirements-dev.in -boto3==1.21.34 - # via - # -c requirements.txt - # -r requirements.in -botocore==1.24.34 +boto3==1.24.62 + # via -r requirements-dev.in +botocore==1.27.62 # via - # -c requirements.txt # boto3 # s3transfer bump2version==1.0.1 @@ -62,15 +54,13 @@ cachetools==5.0.0 # -c requirements.txt # -r requirements.in # google-auth -certifi==2021.10.8 +certifi==2022.6.15 # via # -c requirements.txt # msrest # requests -cffi==1.15.0 - # via - # -c requirements.txt - # cryptography +cffi==1.15.1 + # via cryptography charset-normalizer==2.0.12 # via # -c requirements.txt @@ -92,10 +82,8 @@ commonmark==0.9.1 # rich coverage==6.1.1 # via -r requirements-dev.in -cryptography==36.0.2 - # via - # -c requirements.txt - # azure-storage-blob +cryptography==37.0.4 + # via azure-storage-blob deprecated==1.2.13 # via # -c requirements.txt @@ -118,39 +106,26 @@ frozenlist==1.3.0 # aiosignal ghp-import==2.0.2 # via mkdocs -google-api-core==2.7.1 +google-api-core==2.8.2 # via - # -c requirements.txt # -r requirements-dev.in # google-cloud-core # google-cloud-storage -google-auth==2.6.2 +google-auth==2.11.0 # via - # -c requirements.txt # google-api-core # google-cloud-core # google-cloud-storage -google-cloud-core==2.2.3 - # via - # -c requirements.txt - # google-cloud-storage -google-cloud-storage==2.2.1 - # via - # -c requirements.txt - # -r requirements-dev.in - # -r requirements.in +google-cloud-core==2.3.2 + # via google-cloud-storage +google-cloud-storage==2.5.0 + # via -r requirements-dev.in google-crc32c==1.3.0 - # via - # -c requirements.txt - # google-resumable-media -google-resumable-media==2.3.2 - # via - # -c requirements.txt - # google-cloud-storage -googleapis-common-protos==1.56.0 - # via - # -c requirements.txt - # google-api-core + # via google-resumable-media +google-resumable-media==2.3.3 + # via google-cloud-storage +googleapis-common-protos==1.56.4 + # via google-api-core h11==0.12.0 # via uvicorn humanize==4.0.0 @@ -168,18 +143,15 @@ importlib-metadata==4.8.2 iniconfig==1.1.1 # via pytest isodate==0.6.1 - # via - # -c requirements.txt - # msrest + # via msrest isort==5.10.1 # via -r requirements-dev.in jinja2==3.0.2 # via # mkdocs # mkdocs-material -jmespath==1.0.0 +jmespath==1.0.1 # via - # -c requirements.txt # boto3 # botocore markdown==3.3.4 @@ -201,10 +173,8 @@ mkdocs-material==7.3.6 # via -r requirements-dev.in mkdocs-material-extensions==1.0.3 # via mkdocs-material -msrest==0.6.21 - # via - # -c requirements.txt - # azure-storage-blob +msrest==0.7.1 + # via azure-storage-blob multidict==6.0.2 # via # -c requirements.txt @@ -221,9 +191,7 @@ networkx==2.6.3 nox==2021.10.1 # via -r requirements-dev.in oauthlib==3.2.0 - # via - # -c requirements.txt - # requests-oauthlib + # via requests-oauthlib packaging==21.3 # via # -c requirements.txt @@ -245,9 +213,7 @@ pluggy==1.0.0 # via pytest protobuf==3.20.0 # via - # -c requirements.txt # google-api-core - # google-cloud-storage # googleapis-common-protos psutil==5.8.0 # via memory-profiler @@ -257,19 +223,14 @@ py==1.11.0 # pytest pyasn1==0.4.8 # via - # -c requirements.txt # pyasn1-modules # rsa pyasn1-modules==0.2.8 - # via - # -c requirements.txt - # google-auth + # via google-auth pycodestyle==2.7.0 # via flake8 pycparser==2.21 - # via - # -c requirements.txt - # cffi + # via cffi pydantic==1.9.0 # via # -c requirements.txt @@ -277,7 +238,7 @@ pydantic==1.9.0 # fastapi pyflakes==2.3.1 # via flake8 -pygments==2.11.2 +pygments==2.12.0 # via # -c requirements.txt # mkdocs-material @@ -312,30 +273,25 @@ redis==4.2.2 # via # -c requirements.txt # -r requirements.in -requests==2.27.1 +requests==2.28.1 # via # -c requirements.txt + # -r requirements.in # azure-core # google-api-core # google-cloud-storage # msrest # requests-oauthlib requests-oauthlib==1.3.1 - # via - # -c requirements.txt - # msrest + # via msrest rich==12.2.0 # via # -c requirements.txt # -r requirements.in -rsa==4.8 - # via - # -c requirements.txt - # google-auth -s3transfer==0.5.2 - # via - # -c requirements.txt - # boto3 +rsa==4.9 + # via google-auth +s3transfer==0.6.0 + # via boto3 six==1.16.0 # via # -c requirements.txt @@ -390,7 +346,7 @@ typing-extensions==4.1.1 # azure-core # mypy # pydantic -urllib3==1.26.9 +urllib3==1.26.12 # via # -c requirements.txt # botocore diff --git a/requirements-optional.txt b/requirements-optional.txt index 3703d1f4..61ca7386 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -2,5 +2,11 @@ numpy grpcio -tensorflow==2.8.0 -tensorflow-serving-api==2.8.0 +tensorflow +tensorflow-serving-api +# assets-s3 +boto3 +# assets-gcs +google-cloud-storage +# assets-az +azure-storage-blob \ No newline at end of file diff --git a/requirements.in b/requirements.in index 1daafbd6..843f894c 100644 --- a/requirements.in +++ b/requirements.in @@ -1,15 +1,13 @@ aiohttp asgiref -boto3 cachetools click filelock -google-cloud-storage -azure-storage-blob humanize pydantic python-dateutil redis +requests rich sniffio structlog diff --git a/requirements.txt b/requirements.txt index c20c9963..5db0b8ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,26 +16,10 @@ async-timeout==4.0.2 # redis attrs==21.4.0 # via aiohttp -azure-core==1.23.1 - # via azure-storage-blob -azure-storage-blob==12.11.0 - # via -r requirements.in -boto3==1.21.34 - # via -r requirements.in -botocore==1.24.34 - # via - # boto3 - # s3transfer cachetools==5.0.0 - # via - # -r requirements.in - # google-auth -certifi==2021.10.8 - # via - # msrest - # requests -cffi==1.15.0 - # via cryptography + # via -r requirements.in +certifi==2022.6.15 + # via requests charset-normalizer==2.0.12 # via # aiohttp @@ -44,8 +28,6 @@ click==8.1.2 # via -r requirements.in commonmark==0.9.1 # via rich -cryptography==36.0.2 - # via azure-storage-blob deprecated==1.2.13 # via redis filelock==3.6.0 @@ -54,93 +36,34 @@ frozenlist==1.3.0 # via # aiohttp # aiosignal -google-api-core==2.7.1 - # via - # google-cloud-core - # google-cloud-storage -google-auth==2.6.2 - # via - # google-api-core - # google-cloud-core - # google-cloud-storage -google-cloud-core==2.2.3 - # via google-cloud-storage -google-cloud-storage==2.2.1 - # via -r requirements.in -google-crc32c==1.3.0 - # via google-resumable-media -google-resumable-media==2.3.2 - # via google-cloud-storage -googleapis-common-protos==1.56.0 - # via google-api-core humanize==4.0.0 # via -r requirements.in idna==3.3 # via # requests # yarl -isodate==0.6.1 - # via msrest -jmespath==1.0.0 - # via - # boto3 - # botocore -msrest==0.6.21 - # via azure-storage-blob multidict==6.0.2 # via # aiohttp # yarl -oauthlib==3.2.0 - # via requests-oauthlib packaging==21.3 # via redis -protobuf==3.20.0 - # via - # google-api-core - # google-cloud-storage - # googleapis-common-protos -pyasn1==0.4.8 - # via - # pyasn1-modules - # rsa -pyasn1-modules==0.2.8 - # via google-auth -pycparser==2.21 - # via cffi pydantic==1.9.0 # via -r requirements.in -pygments==2.11.2 +pygments==2.12.0 # via rich pyparsing==3.0.7 # via packaging python-dateutil==2.8.2 - # via - # -r requirements.in - # botocore + # via -r requirements.in redis==4.2.2 # via -r requirements.in -requests==2.27.1 - # via - # azure-core - # google-api-core - # google-cloud-storage - # msrest - # requests-oauthlib -requests-oauthlib==1.3.1 - # via msrest +requests==2.28.1 + # via -r requirements.in rich==12.2.0 # via -r requirements.in -rsa==4.8 - # via google-auth -s3transfer==0.5.2 - # via boto3 six==1.16.0 - # via - # azure-core - # google-auth - # isodate - # python-dateutil + # via python-dateutil sniffio==1.2.0 # via -r requirements.in structlog==21.5.0 @@ -152,12 +75,9 @@ tenacity==8.0.1 typing-extensions==4.1.1 # via # -r requirements.in - # azure-core # pydantic -urllib3==1.26.9 - # via - # botocore - # requests +urllib3==1.26.12 + # via requests wrapt==1.14.0 # via deprecated yarl==1.7.2 diff --git a/setup.cfg b/setup.cfg index f6aa5451..9009eafb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,16 +27,14 @@ packages = find: install_requires = aiohttp asgiref - boto3 cachetools click filelock - google-cloud-storage - azure-storage-blob humanize pydantic python-dateutil redis + requests rich sniffio structlog @@ -57,6 +55,12 @@ cli = api = fastapi uvicorn +assets-s3 = + boto3 +assets-gcs = + google-cloud-storage +assets-az = + azure-storage-blob [options.packages.find] where = . diff --git a/tests/assets/test_retry.py b/tests/assets/test_retry.py index 5fdafd6a..4b60897e 100644 --- a/tests/assets/test_retry.py +++ b/tests/assets/test_retry.py @@ -7,20 +7,28 @@ import requests from tenacity import retry, stop_after_attempt -from modelkit.assets.drivers.retry import RETRY_POLICY +from modelkit.assets.drivers.retry import retry_policy from modelkit.assets.errors import ObjectDoesNotExistError @pytest.mark.parametrize( - "exception, exc_args", + "exception, exc_args, policy", [ - (google.api_core.exceptions.GoogleAPIError, ()), - (botocore.exceptions.ClientError, ({}, "operation_name")), - (requests.exceptions.ChunkedEncodingError, ()), + ( + google.api_core.exceptions.GoogleAPIError, + (), + retry_policy(google.api_core.exceptions.GoogleAPIError), + ), + ( + botocore.exceptions.ClientError, + ({}, "operation_name"), + retry_policy(botocore.exceptions.ClientError), + ), + (requests.exceptions.ChunkedEncodingError, (), retry_policy(None)), ], ) -def test_retry_policy(exception, exc_args): - SHORT_RETRY_POLICY = copy.deepcopy(RETRY_POLICY) +def test_retry_policy(exception, exc_args, policy): + SHORT_RETRY_POLICY = copy.deepcopy(policy) SHORT_RETRY_POLICY["stop"] = stop_after_attempt(2) k = 0 @@ -36,7 +44,7 @@ def some_function(): def test_retry_policy_asset_error(): - SHORT_RETRY_POLICY = copy.deepcopy(RETRY_POLICY) + SHORT_RETRY_POLICY = copy.deepcopy(retry_policy(None)) SHORT_RETRY_POLICY["stop"] = stop_after_attempt(2) k = 0