From 7410516a87455b5e1c307c1e4bf4429843401e52 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:13:31 +0200 Subject: [PATCH] =?UTF-8?q?=E2=AC=86=EF=B8=8FMigration:=20Storage=20(#6599?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aws-library/src/aws_library/s3/_client.py | 11 +- packages/aws-library/tests/test_s3_client.py | 12 +- .../pydantic_networks_extension.py | 7 +- .../src/models_library/api_schemas_storage.py | 25 +-- .../utils/json_serialization.py | 2 + .../src/settings_library/node_ports.py | 11 +- .../src/settings_library/utils_cli.py | 4 +- .../src/simcore_sdk/node_data/data_manager.py | 11 +- .../node_ports_common/filemanager.py | 21 +- scripts/release/monitor_release/settings.py | 4 +- .../computational_sidecar/models.py | 3 +- .../services/resource_tracker_service_runs.py | 4 +- services/storage/requirements/_base.txt | 184 +++++++++++++++++- services/storage/requirements/_test.txt | 15 +- .../simcore_service_storage/application.py | 2 +- .../datcore_adapter/datcore_adapter.py | 11 +- .../datcore_adapter_settings.py | 10 +- .../db_file_meta_data.py | 30 +-- .../simcore_service_storage/db_projects.py | 2 +- .../src/simcore_service_storage/exceptions.py | 10 +- .../handlers_datasets.py | 2 +- .../simcore_service_storage/handlers_files.py | 31 +-- .../handlers_health.py | 17 +- .../handlers_simcore_s3.py | 4 +- .../src/simcore_service_storage/models.py | 94 +++++---- .../src/simcore_service_storage/s3_utils.py | 6 +- .../src/simcore_service_storage/settings.py | 62 ++++-- .../simcore_service_storage/simcore_s3_dsm.py | 66 ++++--- .../simcore_s3_dsm_utils.py | 15 +- .../src/simcore_service_storage/utils.py | 4 +- services/storage/tests/conftest.py | 36 ++-- .../storage/tests/fixtures/data_models.py | 29 ++- .../test__legacy_storage_sdk_compatibility.py | 2 +- services/storage/tests/unit/test_cli.py | 6 +- services/storage/tests/unit/test_dsm.py | 4 +- .../storage/tests/unit/test_dsm_dsmcleaner.py | 34 ++-- .../storage/tests/unit/test_dsm_soft_links.py | 2 +- .../tests/unit/test_handlers_datasets.py | 6 +- .../storage/tests/unit/test_handlers_files.py | 155 ++++++++------- .../unit/test_handlers_files_metadata.py | 20 +- .../tests/unit/test_handlers_health.py | 14 +- .../tests/unit/test_handlers_simcore_s3.py | 58 +++--- services/storage/tests/unit/test_models.py | 4 +- .../storage/tests/unit/test_simcore_s3_dsm.py | 12 +- services/storage/tests/unit/test_utils.py | 23 ++- .../storage/tests/unit/test_utils_handlers.py | 8 +- 46 files changed, 682 insertions(+), 411 deletions(-) diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py index acd9402fda3..38e7c0f9636 100644 --- a/packages/aws-library/src/aws_library/s3/_client.py +++ b/packages/aws-library/src/aws_library/s3/_client.py @@ -13,7 +13,6 @@ from boto3.s3.transfer import TransferConfig from botocore import exceptions as botocore_exc from botocore.client import Config -from common_library.pydantic_networks_extension import AnyUrlLegacy from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart from models_library.basic_types import SHA256Str from pydantic import AnyUrl, ByteSize, TypeAdapter @@ -255,7 +254,7 @@ async def create_single_presigned_download_link( bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int, - ) -> str: + ) -> AnyUrl: # NOTE: ensure the bucket/object exists, this will raise if not await self._client.head_bucket(Bucket=bucket) await self._client.head_object(Bucket=bucket, Key=object_key) @@ -264,12 +263,12 @@ async def create_single_presigned_download_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}" + return TypeAdapter(AnyUrl).validate_python(generated_link) @s3_exception_handler(_logger) async def create_single_presigned_upload_link( self, *, bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int - ) -> str: + ) -> AnyUrl: # NOTE: ensure the bucket/object exists, this will raise if not await self._client.head_bucket(Bucket=bucket) generated_link = await self._client.generate_presigned_url( @@ -277,7 +276,7 @@ async def create_single_presigned_upload_link( Params={"Bucket": bucket, "Key": object_key}, ExpiresIn=expiration_secs, ) - return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}" + return TypeAdapter(AnyUrl).validate_python(generated_link) @s3_exception_handler(_logger) async def create_multipart_upload_links( @@ -474,6 +473,6 @@ def is_multipart(file_size: ByteSize) -> bool: @staticmethod def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl: - return TypeAdapter(AnyUrlLegacy).validate_python( + return TypeAdapter(AnyUrl).validate_python( f"s3://{bucket}/{urllib.parse.quote(object_key)}" ) diff --git a/packages/aws-library/tests/test_s3_client.py b/packages/aws-library/tests/test_s3_client.py index d07075aed30..5e5efc962a5 100644 --- a/packages/aws-library/tests/test_s3_client.py +++ b/packages/aws-library/tests/test_s3_client.py @@ -742,7 +742,7 @@ async def test_create_single_presigned_upload_link( create_file_of_size: Callable[[ByteSize], Path], default_expiration_time_seconds: int, upload_to_presigned_link: Callable[ - [Path, str, S3BucketName, S3ObjectKey], Awaitable[None] + [Path, AnyUrl, S3BucketName, S3ObjectKey], Awaitable[None] ], ): file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1Mib")) @@ -1289,12 +1289,16 @@ def test_is_multipart(file_size: ByteSize, expected_multipart: bool): ( "some-bucket", "an/object/separate/by/slashes", - "s3://some-bucket/an/object/separate/by/slashes", + TypeAdapter(AnyUrl).validate_python( + "s3://some-bucket/an/object/separate/by/slashes" + ), ), ( "some-bucket", "an/object/separate/by/slashes-?/3#$", - r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24", + TypeAdapter(AnyUrl).validate_python( + r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24" + ), ), ], ) @@ -1302,7 +1306,7 @@ def test_compute_s3_url( bucket: S3BucketName, object_key: S3ObjectKey, expected_s3_url: AnyUrl ): assert ( - str(SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key)) + SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key) == expected_s3_url ) diff --git a/packages/common-library/src/common_library/pydantic_networks_extension.py b/packages/common-library/src/common_library/pydantic_networks_extension.py index 79c5da906b1..1e85a741760 100644 --- a/packages/common-library/src/common_library/pydantic_networks_extension.py +++ b/packages/common-library/src/common_library/pydantic_networks_extension.py @@ -1,6 +1,6 @@ from typing import Annotated, TypeAlias -from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, HttpUrl +from pydantic import AfterValidator, AnyHttpUrl, HttpUrl from pydantic_core import Url @@ -8,11 +8,6 @@ def _strip_last_slash(url: Url) -> str: return f"{url}".rstrip("/") -AnyUrlLegacy: TypeAlias = Annotated[ - AnyUrl, - AfterValidator(_strip_last_slash), -] - AnyHttpUrlLegacy: TypeAlias = Annotated[ AnyHttpUrl, AfterValidator(_strip_last_slash), diff --git a/packages/models-library/src/models_library/api_schemas_storage.py b/packages/models-library/src/models_library/api_schemas_storage.py index bd0185a9173..54c5e28d43d 100644 --- a/packages/models-library/src/models_library/api_schemas_storage.py +++ b/packages/models-library/src/models_library/api_schemas_storage.py @@ -8,7 +8,7 @@ from datetime import datetime from enum import Enum -from typing import Annotated, Any, TypeAlias +from typing import Annotated, Any, Literal, Self, TypeAlias from uuid import UUID from pydantic import ( @@ -107,6 +107,10 @@ class DatasetMetaDataGet(BaseModel): ) +UNDEFINED_SIZE_TYPE: TypeAlias = Literal[-1] +UNDEFINED_SIZE: UNDEFINED_SIZE_TYPE = -1 + + # /locations/{location_id}/files/metadata: # /locations/{location_id}/files/{file_id}/metadata: class FileMetaDataGet(BaseModel): @@ -130,8 +134,8 @@ class FileMetaDataGet(BaseModel): ) created_at: datetime last_modified: datetime - file_size: ByteSize | int = Field( - default=-1, description="File size in bytes (-1 means invalid)" + file_size: UNDEFINED_SIZE_TYPE | ByteSize = Field( + default=UNDEFINED_SIZE, description="File size in bytes (-1 means invalid)" ) entity_tag: ETag | None = Field( default=None, @@ -149,7 +153,7 @@ class FileMetaDataGet(BaseModel): ) model_config = ConfigDict( - extra="forbid", + extra="ignore", from_attributes=True, json_schema_extra={ "examples": [ @@ -312,19 +316,18 @@ class FoldersBody(BaseModel): nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict) @model_validator(mode="after") - @classmethod - def ensure_consistent_entries(cls, values): - source_node_keys = (NodeID(n) for n in values["source"].get("workbench", {})) - if set(source_node_keys) != set(values["nodes_map"].keys()): + def ensure_consistent_entries(self) -> Self: + source_node_keys = (NodeID(n) for n in self.source.get("workbench", {})) + if set(source_node_keys) != set(self.nodes_map.keys()): msg = "source project nodes do not fit with nodes_map entries" raise ValueError(msg) destination_node_keys = ( - NodeID(n) for n in values["destination"].get("workbench", {}) + NodeID(n) for n in self.destination.get("workbench", {}) ) - if set(destination_node_keys) != set(values["nodes_map"].values()): + if set(destination_node_keys) != set(self.nodes_map.values()): msg = "destination project nodes do not fit with nodes_map values" raise ValueError(msg) - return values + return self class SoftCopyBody(BaseModel): diff --git a/packages/models-library/src/models_library/utils/json_serialization.py b/packages/models-library/src/models_library/utils/json_serialization.py index 9bd0abd5ef6..ef11a2640cd 100644 --- a/packages/models-library/src/models_library/utils/json_serialization.py +++ b/packages/models-library/src/models_library/utils/json_serialization.py @@ -24,6 +24,7 @@ import orjson from pydantic import NameEmail, SecretBytes, SecretStr +from pydantic_core import Url from pydantic_extra_types.color import Color @@ -84,6 +85,7 @@ def decimal_encoder(dec_value: Decimal) -> int | float: SecretBytes: str, SecretStr: str, set: list, + Url: str, UUID: str, } diff --git a/packages/settings-library/src/settings_library/node_ports.py b/packages/settings-library/src/settings_library/node_ports.py index 522fcdd0991..8498c882e4d 100644 --- a/packages/settings-library/src/settings_library/node_ports.py +++ b/packages/settings-library/src/settings_library/node_ports.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import Final +from typing import Final, Self from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator @@ -22,14 +22,13 @@ def auth_required(self) -> bool: return self.STORAGE_USERNAME is not None and self.STORAGE_PASSWORD is not None @model_validator(mode="after") - @classmethod - def _validate_auth_fields(cls, values): - username = values.STORAGE_USERNAME - password = values.STORAGE_PASSWORD + def _validate_auth_fields(self) -> Self: + username = self.STORAGE_USERNAME + password = self.STORAGE_PASSWORD if (username is None) != (password is None): msg = f"Both {username=} and {password=} must be either set or unset!" raise ValueError(msg) - return values + return self class NodePortsSettings(BaseCustomSettings): diff --git a/packages/settings-library/src/settings_library/utils_cli.py b/packages/settings-library/src/settings_library/utils_cli.py index 0311ed28d7b..e3addbc6130 100644 --- a/packages/settings-library/src/settings_library/utils_cli.py +++ b/packages/settings-library/src/settings_library/utils_cli.py @@ -2,6 +2,7 @@ import logging import os from collections.abc import Callable +from enum import Enum from pprint import pformat from typing import Any @@ -64,7 +65,8 @@ def print_as_envfile( if verbose and field.description: typer.echo(f"# {field.description}") - + if isinstance(value, Enum): + value = value.value typer.echo(f"{name}={value}") diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py index 7579c3eeb0c..b288a295db9 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, StorageFileID from models_library.users import UserID -from pydantic import ByteSize, TypeAdapter +from pydantic import TypeAdapter from servicelib.archiving_utils import unarchive_dir from servicelib.logging_utils import log_context from servicelib.progress_bar import ProgressBarData @@ -184,15 +184,6 @@ async def _delete_legacy_archive( ) -async def get_remote_size( - *, user_id: UserID, project_id: ProjectID, node_uuid: NodeID, source_path: Path -) -> ByteSize: - s3_object = __create_s3_object_key(project_id, node_uuid, source_path) - return await filemanager.get_path_size( - user_id=user_id, store_id=SIMCORE_LOCATION, s3_object=s3_object - ) - - async def push( user_id: UserID, project_id: ProjectID, diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py index f3e2587fab7..b7180877037 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py @@ -15,7 +15,7 @@ from models_library.basic_types import IDStr, SHA256Str from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, TypeAdapter, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.file_utils import create_sha256_checksum from servicelib.progress_bar import ProgressBarData from settings_library.aws_s3_cli import AwsS3CliSettings @@ -189,7 +189,9 @@ async def download_path_from_s3( aws_s3_cli_settings, progress_bar, local_directory_path=local_path, - download_s3_link=TypeAdapter(AnyUrl).validate_python(f"{download_link}"), + download_s3_link=TypeAdapter(AnyUrl).validate_python( + f"{download_link}" + ), ) elif r_clone_settings: await r_clone.sync_s3_to_local( @@ -572,21 +574,6 @@ async def get_file_metadata( ) -async def get_path_size( - user_id: UserID, - store_id: LocationID, - s3_object: StorageFileID, - client_session: ClientSession | None = None, -) -> ByteSize: - file_metadata: FileMetaDataGet = await _get_file_meta_data( - user_id=user_id, - store_id=store_id, - s3_object=s3_object, - client_session=client_session, - ) - return ByteSize(file_metadata.file_size) - - async def delete_file( user_id: UserID, store_id: LocationID, diff --git a/scripts/release/monitor_release/settings.py b/scripts/release/monitor_release/settings.py index ba4f9c7e1d2..947732f2ceb 100644 --- a/scripts/release/monitor_release/settings.py +++ b/scripts/release/monitor_release/settings.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Final +from typing import Final, Self from dotenv import load_dotenv from pydantic import BaseModel, Field, HttpUrl, TypeAdapter, model_validator @@ -47,7 +47,7 @@ class ReleaseSettings(BaseSettings): model_config = SettingsConfigDict(extra="ignore") @model_validator(mode="after") - def deduce_portainer_url(self): + def deduce_portainer_url(self) -> Self: self.portainer_url = TypeAdapter(HttpUrl).validate_python( f"https://{self.PORTAINER_DOMAIN}" ) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py index ee270aeb888..c505329af50 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/computational_sidecar/models.py @@ -1,4 +1,5 @@ import re +from typing import Self from models_library.basic_regex import SIMPLE_VERSION_RE from models_library.services import ServiceMetaDataPublished @@ -49,7 +50,7 @@ class ContainerHostConfig(BaseModel): ) @model_validator(mode="after") - def ensure_memory_swap_is_not_unlimited(self) -> "ContainerHostConfig": + def ensure_memory_swap_is_not_unlimited(self) -> Self: if self.memory_swap is None: self.memory_swap = self.memory diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index a02f4ecd646..e5145f8f6bd 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -177,12 +177,12 @@ async def export_service_runs( ) # Create presigned S3 link - generated_url: str = await s3_client.create_single_presigned_download_link( + generated_url = await s3_client.create_single_presigned_download_link( bucket=s3_bucket_name, object_key=s3_object_key, expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC, ) - return generated_url + return f"{generated_url}" async def get_osparc_credits_aggregated_usages_page( diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index d5a191d6e58..c2766e7dc10 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -29,17 +29,30 @@ aiofiles==23.2.1 # aioboto3 aiohttp==3.9.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_aiohttp.in @@ -61,6 +74,8 @@ aiosignal==1.3.1 # via aiohttp alembic==1.13.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -97,17 +112,30 @@ botocore-stubs==1.34.69 # via types-aiobotocore certifi==2024.2.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # requests @@ -155,17 +183,30 @@ isodate==0.6.1 # via openapi-core jinja2==3.1.3 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiohttp-swagger @@ -195,17 +236,30 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator mako==1.3.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # alembic @@ -302,17 +356,30 @@ opentelemetry-util-http==0.47b0 # opentelemetry-instrumentation-requests orjson==3.10.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in @@ -341,35 +408,80 @@ psycopg2-binary==2.9.9 # via # aiopg # sqlalchemy -pydantic==1.10.14 +pydantic==2.9.2 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.6.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -380,19 +492,34 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.1 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -401,17 +528,30 @@ pyyaml==6.0.1 # jsonschema-path redis==5.0.4 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in @@ -461,17 +601,30 @@ sniffio==1.3.1 # via anyio sqlalchemy==1.4.52 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -519,6 +672,7 @@ typing-extensions==4.10.0 # faststream # opentelemetry-sdk # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 @@ -526,33 +680,59 @@ typing-extensions==4.10.0 # types-aiobotocore-ssm ujson==5.9.0 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # aiohttp-swagger urllib3==2.0.7 # via + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # botocore diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index f0132fe4c7c..2915015ad89 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -11,6 +11,10 @@ aiosignal==1.3.1 # via # -c requirements/_base.txt # aiohttp +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto attrs==23.2.0 @@ -193,11 +197,15 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.4 # via moto pytest==8.3.3 @@ -238,7 +246,9 @@ python-dateutil==2.9.0.post0 # pandas # simcore-service-storage-sdk python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pytz==2024.2 # via pandas pyyaml==6.0.1 @@ -319,6 +329,7 @@ typing-extensions==4.10.0 # cfn-lint # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs tzdata==2024.2 # via pandas diff --git a/services/storage/src/simcore_service_storage/application.py b/services/storage/src/simcore_service_storage/application.py index 7ad35793bd7..77b207a513a 100644 --- a/services/storage/src/simcore_service_storage/application.py +++ b/services/storage/src/simcore_service_storage/application.py @@ -44,7 +44,7 @@ def create(settings: Settings) -> web.Application: _logger.debug( "Initializing app with settings:\n%s", - settings.json(indent=2, sort_keys=True), + settings.model_dump_json(indent=2), ) app = create_safe_application(None) diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py index 5e1f4cff09c..8fb9a162c52 100644 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py +++ b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter.py @@ -1,4 +1,3 @@ -import asyncio import logging from collections.abc import Callable from math import ceil @@ -9,7 +8,7 @@ from aiohttp.client import ClientSession from models_library.api_schemas_storage import DatCoreDatasetName from models_library.users import UserID -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from servicelib.aiohttp.application_keys import APP_CONFIG_KEY from servicelib.aiohttp.client_session import get_client_session from servicelib.utils import logged_gather @@ -73,7 +72,7 @@ async def _request( except aiohttp.ClientResponseError as exc: raise _DatcoreAdapterResponseError(status=exc.status, reason=f"{exc}") from exc - except asyncio.TimeoutError as exc: + except TimeoutError as exc: msg = f"datcore-adapter server timed-out: {exc}" raise DatcoreAdapterTimeoutError(msg) from exc @@ -122,7 +121,7 @@ async def check_service_health(app: web.Application) -> bool: session: ClientSession = get_client_session(app) try: await session.get(url, raise_for_status=True) - except (asyncio.TimeoutError, aiohttp.ClientError): + except (TimeoutError, aiohttp.ClientError): return False return True @@ -187,7 +186,7 @@ async def list_all_files_metadatas_in_dataset( ), ) return [ - FileMetaData.construct( + FileMetaData.model_construct( file_uuid=d["path"], location_id=DATCORE_ID, location=DATCORE_STR, @@ -229,7 +228,7 @@ async def get_file_download_presigned_link( dict[str, Any], await _request(app, api_key, api_secret, "GET", f"/files/{file_id}"), ) - url: AnyUrl = parse_obj_as(AnyUrl, file_download_data["link"]) + url: AnyUrl = TypeAdapter(AnyUrl).validate_python(file_download_data["link"]) return url diff --git a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py index c1322590eaa..71905e42381 100644 --- a/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py +++ b/services/storage/src/simcore_service_storage/datcore_adapter/datcore_adapter_settings.py @@ -1,24 +1,24 @@ from functools import cached_property from models_library.basic_types import PortInt, VersionTag -from pydantic import AnyHttpUrl, Field +from pydantic import AnyHttpUrl, Field, TypeAdapter from settings_library.base import BaseCustomSettings class DatcoreAdapterSettings(BaseCustomSettings): DATCORE_ADAPTER_ENABLED: bool = True DATCORE_ADAPTER_HOST: str = "datcore-adapter" - DATCORE_ADAPTER_PORT: PortInt = PortInt(8000) + DATCORE_ADAPTER_PORT: PortInt = TypeAdapter(PortInt).validate_python(8000) DATCORE_ADAPTER_VTAG: VersionTag = Field( "v0", description="Datcore-adapter service API's version tag" ) @cached_property def endpoint(self) -> str: - endpoint: str = AnyHttpUrl.build( + endpoint = AnyHttpUrl.build( scheme="http", host=self.DATCORE_ADAPTER_HOST, - port=f"{self.DATCORE_ADAPTER_PORT}", + port=self.DATCORE_ADAPTER_PORT, path=f"/{self.DATCORE_ADAPTER_VTAG}", ) - return endpoint + return f"{endpoint}" diff --git a/services/storage/src/simcore_service_storage/db_file_meta_data.py b/services/storage/src/simcore_service_storage/db_file_meta_data.py index 21c519e5483..b742449ee00 100644 --- a/services/storage/src/simcore_service_storage/db_file_meta_data.py +++ b/services/storage/src/simcore_service_storage/db_file_meta_data.py @@ -32,7 +32,9 @@ async def upsert( ) -> FileMetaDataAtDB: # NOTE: upsert file_meta_data, if the file already exists, we update the whole row # so we get the correct time stamps - fmd_db = FileMetaDataAtDB.from_orm(fmd) if isinstance(fmd, FileMetaData) else fmd + fmd_db = ( + FileMetaDataAtDB.model_validate(fmd) if isinstance(fmd, FileMetaData) else fmd + ) insert_statement = pg_insert(file_meta_data).values(**jsonable_encoder(fmd_db)) on_update_statement = insert_statement.on_conflict_do_update( index_elements=[file_meta_data.c.file_id], set_=jsonable_encoder(fmd_db) @@ -40,11 +42,11 @@ async def upsert( result = await conn.execute(on_update_statement) row = await result.first() assert row # nosec - return FileMetaDataAtDB.from_orm(row) + return FileMetaDataAtDB.model_validate(row) async def insert(conn: SAConnection, fmd: FileMetaData) -> FileMetaDataAtDB: - fmd_db = FileMetaDataAtDB.from_orm(fmd) + fmd_db = FileMetaDataAtDB.model_validate(fmd) result = await conn.execute( file_meta_data.insert() .values(jsonable_encoder(fmd_db)) @@ -52,7 +54,7 @@ async def insert(conn: SAConnection, fmd: FileMetaData) -> FileMetaDataAtDB: ) row = await result.first() assert row # nosec - return FileMetaDataAtDB.from_orm(row) + return FileMetaDataAtDB.model_validate(row) async def get(conn: SAConnection, file_id: SimcoreS3FileID) -> FileMetaDataAtDB: @@ -60,7 +62,7 @@ async def get(conn: SAConnection, file_id: SimcoreS3FileID) -> FileMetaDataAtDB: query=sa.select(file_meta_data).where(file_meta_data.c.file_id == file_id) ) if row := await result.first(): - return FileMetaDataAtDB.from_orm(row) + return FileMetaDataAtDB.model_validate(row) raise FileMetaDataNotFoundError(file_id=file_id) @@ -83,9 +85,11 @@ def _list_filter_with_partial_file_id_stmt( conditions.append( sa.or_( file_meta_data.c.user_id == f"{user_id}", - file_meta_data.c.project_id.in_(f"{_}" for _ in project_ids) - if project_ids - else False, + ( + file_meta_data.c.project_id.in_(f"{_}" for _ in project_ids) + if project_ids + else False + ), ) ) @@ -130,7 +134,9 @@ async def list_filter_with_partial_file_id( offset=offset, ) - return [FileMetaDataAtDB.from_orm(row) async for row in await conn.execute(stmt)] + return [ + FileMetaDataAtDB.model_validate(row) async for row in await conn.execute(stmt) + ] async def list_fmds( @@ -158,7 +164,9 @@ async def list_fmds( ) ) - return [FileMetaDataAtDB.from_orm(row) async for row in await conn.execute(stmt)] + return [ + FileMetaDataAtDB.model_validate(row) async for row in await conn.execute(stmt) + ] async def total(conn: SAConnection) -> int: @@ -177,7 +185,7 @@ async def list_valid_uploads( file_meta_data.c.upload_expires_at == None # lgtm [py/test-equals-none] ) ): - fmd_at_db = FileMetaDataAtDB.from_orm(row) + fmd_at_db = FileMetaDataAtDB.model_validate(row) yield fmd_at_db diff --git a/services/storage/src/simcore_service_storage/db_projects.py b/services/storage/src/simcore_service_storage/db_projects.py index a8bf48db80e..dc680c491ee 100644 --- a/services/storage/src/simcore_service_storage/db_projects.py +++ b/services/storage/src/simcore_service_storage/db_projects.py @@ -22,7 +22,7 @@ async def list_valid_projects_in( ) ): with suppress(ValidationError): - yield ProjectAtDB.from_orm(row) + yield ProjectAtDB.model_validate(row) async def project_exists( diff --git a/services/storage/src/simcore_service_storage/exceptions.py b/services/storage/src/simcore_service_storage/exceptions.py index d41c6d16d75..937a3afdd06 100644 --- a/services/storage/src/simcore_service_storage/exceptions.py +++ b/services/storage/src/simcore_service_storage/exceptions.py @@ -1,37 +1,31 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class StorageRuntimeError(PydanticErrorMixin, RuntimeError): +class StorageRuntimeError(OsparcErrorMixin, RuntimeError): ... class DatabaseAccessError(StorageRuntimeError): - code = "database.access_error" msg_template: str = "Unexpected error while accessing database backend" class FileMetaDataNotFoundError(DatabaseAccessError): - code = "filemetadata.not_found_error" msg_template: str = "The file meta data for {file_id} was not found" class FileAccessRightError(DatabaseAccessError): - code = "file.access_right_error" msg_template: str = "Insufficient access rights to {access_right} data {file_id}" class ProjectAccessRightError(DatabaseAccessError): - code = "file.access_right_error" msg_template: str = ( "Insufficient access rights to {access_right} project {project_id}" ) class ProjectNotFoundError(DatabaseAccessError): - code = "project.not_found_error" msg_template: str = "Project {project_id} was not found" class LinkAlreadyExistsError(DatabaseAccessError): - code = "link.already_exists_error" msg_template: str = "The link {file_id} already exists" diff --git a/services/storage/src/simcore_service_storage/handlers_datasets.py b/services/storage/src/simcore_service_storage/handlers_datasets.py index 4041ba7e509..69b345247e2 100644 --- a/services/storage/src/simcore_service_storage/handlers_datasets.py +++ b/services/storage/src/simcore_service_storage/handlers_datasets.py @@ -69,6 +69,6 @@ async def get_files_metadata_dataset(request: web.Request) -> web.Response: expand_dirs=query_params.expand_dirs, ) return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet.from_orm(d)) for d in data]}, + {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, dumps=json_dumps, ) diff --git a/services/storage/src/simcore_service_storage/handlers_files.py b/services/storage/src/simcore_service_storage/handlers_files.py index 87cb60d5829..b7ad6ebb007 100644 --- a/services/storage/src/simcore_service_storage/handlers_files.py +++ b/services/storage/src/simcore_service_storage/handlers_files.py @@ -18,7 +18,7 @@ ) from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.utils.json_serialization import json_dumps -from pydantic import AnyUrl, ByteSize, parse_obj_as +from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -72,7 +72,7 @@ async def get_files_metadata(request: web.Request) -> web.Response: project_id=query_params.project_id, ) return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet.from_orm(d)) for d in data]}, + {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, dumps=json_dumps, ) @@ -87,7 +87,7 @@ async def get_file_metadata(request: web.Request) -> web.Response: ) path_params = parse_request_path_parameters_as(FilePathParams, request) log.debug( - "received call to get_files_metadata_dataset with %s", + "received call to get_file_metadata_dataset with %s", f"{path_params=}, {query_params=}", ) @@ -134,7 +134,10 @@ async def get_file_metadata(request: web.Request) -> web.Response: dumps=json_dumps, ) - return jsonable_encoder(FileMetaDataGet.from_orm(data)) # type: ignore[no-any-return] # middleware takes care of enveloping + return web.json_response( + {"data": jsonable_encoder(FileMetaDataGet(**data.model_dump()))}, + dumps=json_dumps, + ) @routes.get( @@ -177,7 +180,8 @@ async def upload_file(request: web.Request) -> web.Response: - client calls complete_upload handle which will reconstruct the file on S3 backend - client waits for completion to finish and then the file is accessible on S3 backend - Use-case v1: if query.file_size is not defined, returns a PresignedLink model (backward compatibility) + + Use-case v1: query.file_size is not defined, returns a PresignedLink model (backward compatibility) Use-case v1.1: if query.link_type=presigned or None, returns a presigned link (limited to a single 5GB file) Use-case v1.2: if query.link_type=s3, returns a s3 direct link (limited to a single 5TB file) @@ -205,10 +209,12 @@ async def upload_file(request: web.Request) -> web.Response: is_directory=query_params.is_directory, sha256_checksum=query_params.sha256_checksum, ) - if query_params.file_size is None and not query_params.is_directory: + if query_params.is_v1_upload: # return v1 response assert len(links.urls) == 1 # nosec - response = {"data": {"link": jsonable_encoder(links.urls[0], by_alias=True)}} + response = { + "data": {"link": jsonable_encoder(f"{links.urls[0]}", by_alias=True)} + } log.debug("Returning v1 response: %s", response) return web.json_response(response, dumps=json_dumps) @@ -233,11 +239,8 @@ async def upload_file(request: web.Request) -> web.Response: chunk_size=links.chunk_size, urls=links.urls, links=FileUploadLinks( - abort_upload=parse_obj_as(AnyUrl, f"{abort_url}"), - complete_upload=parse_obj_as( - AnyUrl, - f"{complete_url}", - ), + abort_upload=TypeAdapter(AnyUrl).validate_python(f"{abort_url}"), + complete_upload=TypeAdapter(AnyUrl).validate_python(f"{complete_url}"), ), ) log.debug("returning v2 response: %s", v2_response) @@ -306,7 +309,7 @@ async def complete_upload_file(request: web.Request) -> web.Response: complete_task_state_url = f"{request.url.scheme}://{ip_addr}:{port}{route}" response = FileUploadCompleteResponse( links=FileUploadCompleteLinks( - state=parse_obj_as(AnyUrl, complete_task_state_url) + state=TypeAdapter(AnyUrl).validate_python(complete_task_state_url) ) ) return web.json_response( @@ -408,4 +411,4 @@ async def copy_as_soft_link(request: web.Request): query_params.user_id, path_params.file_id, body.link_id ) - return jsonable_encoder(FileMetaDataGet.from_orm(file_link)) + return jsonable_encoder(FileMetaDataGet(**file_link.model_dump())) diff --git a/services/storage/src/simcore_service_storage/handlers_health.py b/services/storage/src/simcore_service_storage/handlers_health.py index eb94feb9bb4..5be7fda4e23 100644 --- a/services/storage/src/simcore_service_storage/handlers_health.py +++ b/services/storage/src/simcore_service_storage/handlers_health.py @@ -31,13 +31,12 @@ async def get_health(request: web.Request) -> web.Response: assert request # nosec return web.json_response( { - "data": HealthCheck.parse_obj( - { - "name": PROJECT_NAME, - "version": f"{VERSION}", - "api_version": API_VERSION, - } - ).dict(**RESPONSE_MODEL_POLICY) + "data": HealthCheck( + name=PROJECT_NAME, + version=f"{VERSION}", + api_version=API_VERSION, + status=None, + ).model_dump(**RESPONSE_MODEL_POLICY) }, dumps=json_dumps, ) @@ -69,7 +68,7 @@ async def get_status(request: web.Request) -> web.Response: "connected" if await is_pg_responsive(request.app) else "failed" ) - status = AppStatusCheck.parse_obj( + status = AppStatusCheck.model_validate( { "app_name": PROJECT_NAME, "version": f"{VERSION}", @@ -84,5 +83,5 @@ async def get_status(request: web.Request) -> web.Response: ) return web.json_response( - {"data": status.dict(exclude_unset=True)}, dumps=json_dumps + {"data": status.model_dump(exclude_unset=True)}, dumps=json_dumps ) diff --git a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py index 0f8e52fa7fc..47dd35bc6bc 100644 --- a/services/storage/src/simcore_service_storage/handlers_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/handlers_simcore_s3.py @@ -53,7 +53,7 @@ async def get_or_create_temporary_s3_access(request: web.Request) -> web.Respons s3_settings: S3Settings = await sts.get_or_create_temporary_token_for_user( request.app, query_params.user_id ) - return web.json_response({"data": s3_settings.dict()}, dumps=json_dumps) + return web.json_response({"data": s3_settings.model_dump()}, dumps=json_dumps) async def _copy_folders_from_project( @@ -160,6 +160,6 @@ async def search_files(request: web.Request) -> web.Response: ) return web.json_response( - {"data": [jsonable_encoder(FileMetaDataGet.from_orm(d)) for d in data]}, + {"data": [jsonable_encoder(FileMetaDataGet(**d.model_dump())) for d in data]}, dumps=json_dumps, ) diff --git a/services/storage/src/simcore_service_storage/models.py b/services/storage/src/simcore_service_storage/models.py index d05099edd06..2e21fc2483c 100644 --- a/services/storage/src/simcore_service_storage/models.py +++ b/services/storage/src/simcore_service_storage/models.py @@ -1,11 +1,14 @@ import datetime import urllib.parse from dataclasses import dataclass -from typing import Final, Literal, NamedTuple +from typing import Any, Literal, NamedTuple from uuid import UUID +import arrow from aws_library.s3 import UploadID from models_library.api_schemas_storage import ( + UNDEFINED_SIZE, + UNDEFINED_SIZE_TYPE, DatasetMetaDataGet, ETag, FileMetaDataGet, @@ -31,16 +34,14 @@ AnyUrl, BaseModel, ByteSize, - Extra, + ConfigDict, Field, - parse_obj_as, - root_validator, - validate_arguments, - validator, + TypeAdapter, + field_validator, + model_validator, + validate_call, ) -UNDEFINED_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, -1) - class DatasetMetaData(DatasetMetaDataGet): ... @@ -64,7 +65,7 @@ class FileMetaDataAtDB(BaseModel): user_id: UserID created_at: datetime.datetime file_id: SimcoreS3FileID - file_size: ByteSize + file_size: UNDEFINED_SIZE_TYPE | ByteSize last_modified: datetime.datetime entity_tag: ETag | None = None is_soft_link: bool @@ -73,9 +74,7 @@ class FileMetaDataAtDB(BaseModel): is_directory: bool sha256_checksum: SHA256Str | None = None - class Config: - orm_mode = True - extra = Extra.forbid + model_config = ConfigDict(from_attributes=True, extra="forbid") class FileMetaData(FileMetaDataGet): @@ -91,7 +90,7 @@ class FileMetaData(FileMetaDataGet): sha256_checksum: SHA256Str | None @classmethod - @validate_arguments + @validate_call def from_simcore_node( cls, user_id: UserID, @@ -103,7 +102,7 @@ def from_simcore_node( **file_meta_data_kwargs, ): parts = file_id.split("/") - now = datetime.datetime.utcnow() + now = arrow.utcnow().datetime fmd_kwargs = { "file_uuid": file_id, "location_id": location_id, @@ -113,9 +112,15 @@ def from_simcore_node( "file_name": parts[-1], "user_id": user_id, "project_id": ( - parse_obj_as(ProjectID, parts[0]) if is_uuid(parts[0]) else None + TypeAdapter(ProjectID).validate_python(parts[0]) + if is_uuid(parts[0]) + else None + ), + "node_id": ( + TypeAdapter(NodeID).validate_python(parts[1]) + if is_uuid(parts[1]) + else None ), - "node_id": parse_obj_as(NodeID, parts[1]) if is_uuid(parts[1]) else None, "file_id": file_id, "created_at": now, "last_modified": now, @@ -128,7 +133,7 @@ def from_simcore_node( "is_directory": False, } fmd_kwargs.update(**file_meta_data_kwargs) - return cls.parse_obj(fmd_kwargs) + return cls.model_validate(fmd_kwargs) @dataclass @@ -139,10 +144,7 @@ class UploadLinks: class StorageQueryParamsBase(BaseModel): user_id: UserID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict(populate_by_name=True, extra="forbid") class FilesMetadataDatasetQueryParams(StorageQueryParamsBase): @@ -163,9 +165,9 @@ class SyncMetadataQueryParams(BaseModel): class FileDownloadQueryParams(StorageQueryParamsBase): link_type: LinkType = LinkType.PRESIGNED - @validator("link_type", pre=True) + @field_validator("link_type", mode="before") @classmethod - def convert_from_lower_case(cls, v): + def convert_from_lower_case(cls, v: str) -> str: if v is not None: return f"{v}".upper() return v @@ -173,26 +175,39 @@ def convert_from_lower_case(cls, v): class FileUploadQueryParams(StorageQueryParamsBase): link_type: LinkType = LinkType.PRESIGNED - file_size: ByteSize | None + file_size: ByteSize | None = None # NOTE: in old legacy services this might happen is_directory: bool = False sha256_checksum: SHA256Str | None = None - @validator("link_type", pre=True) + @field_validator("link_type", mode="before") @classmethod - def convert_from_lower_case(cls, v): + def convert_from_lower_case(cls, v: str) -> str: if v is not None: return f"{v}".upper() return v - @root_validator() + @model_validator(mode="before") @classmethod - def when_directory_force_link_type_and_file_size(cls, values): - if values["is_directory"] is True: + def when_directory_force_link_type_and_file_size(cls, data: Any) -> Any: + assert isinstance(data, dict) + + if bool(data.get("is_directory", False)) is True: # sets directory size by default to undefined - values["file_size"] = UNDEFINED_SIZE + if int(data.get("file_size", -1)) < 0: + data["file_size"] = None # only 1 link will be returned manged by the uploader - values["link_type"] = LinkType.S3 - return values + data["link_type"] = LinkType.S3.value + return data + + @property + def is_v1_upload(self) -> bool: + """This returns True if the query params are missing the file_size query parameter, which was the case in the legacy services that have an old version of simcore-sdk + v1 rationale: + - client calls this handler, which returns a single link (either direct S3 or presigned) to the S3 backend + - client uploads the file + - storage relies on lazy update to find if the file is finished uploaded (when client calls get_file_meta_data, or if the dsm_cleaner goes over it after the upload time is expired) + """ + return self.file_size is None and self.is_directory is False class DeleteFolderQueryParams(StorageQueryParamsBase): @@ -211,17 +226,14 @@ class SearchFilesQueryParams(StorageQueryParamsBase): ) offset: int = Field(default=0, ge=0, description="Page offset") - _empty_is_none = validator("startswith", allow_reuse=True, pre=True)( + _empty_is_none = field_validator("startswith", mode="before")( empty_str_to_none_pre_validator ) class LocationPathParams(BaseModel): location_id: LocationID - - class Config: - allow_population_by_field_name = True - extra = Extra.forbid + model_config = ConfigDict(populate_by_name=True, extra="forbid") class FilesMetadataDatasetPathParams(LocationPathParams): @@ -231,9 +243,9 @@ class FilesMetadataDatasetPathParams(LocationPathParams): class FilePathParams(LocationPathParams): file_id: StorageFileID - @validator("file_id", pre=True) + @field_validator("file_id", mode="before") @classmethod - def unquote(cls, v): + def unquote(cls, v: str) -> str: if v is not None: return urllib.parse.unquote(f"{v}") return v @@ -250,9 +262,9 @@ class SimcoreS3FoldersParams(BaseModel): class CopyAsSoftLinkParams(BaseModel): file_id: StorageFileID - @validator("file_id", pre=True) + @field_validator("file_id", mode="before") @classmethod - def unquote(cls, v): + def unquote(cls, v: str) -> str: if v is not None: return urllib.parse.unquote(f"{v}") return v diff --git a/services/storage/src/simcore_service_storage/s3_utils.py b/services/storage/src/simcore_service_storage/s3_utils.py index 641bebf4e64..f40d33d531f 100644 --- a/services/storage/src/simcore_service_storage/s3_utils.py +++ b/services/storage/src/simcore_service_storage/s3_utils.py @@ -2,7 +2,7 @@ from collections import defaultdict from dataclasses import dataclass, field -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.aiohttp.long_running_tasks.server import ( ProgressMessage, ProgressPercent, @@ -55,7 +55,7 @@ def finalize_transfer(self) -> None: def copy_transfer_cb(self, total_bytes_copied: int, *, file_name: str) -> None: _logger.debug( "Copied %s of %s", - parse_obj_as(ByteSize, total_bytes_copied).human_readable(), + TypeAdapter(ByteSize).validate_python(total_bytes_copied).human_readable(), file_name, ) self._file_total_bytes_copied[file_name] = total_bytes_copied @@ -66,7 +66,7 @@ def copy_transfer_cb(self, total_bytes_copied: int, *, file_name: str) -> None: def upload_transfer_cb(self, bytes_transferred: int, *, file_name: str) -> None: _logger.debug( "Uploaded %s of %s", - parse_obj_as(ByteSize, bytes_transferred).human_readable(), + TypeAdapter(ByteSize).validate_python(bytes_transferred).human_readable(), file_name, ) self._file_total_bytes_copied[file_name] += bytes_transferred diff --git a/services/storage/src/simcore_service_storage/settings.py b/services/storage/src/simcore_service_storage/settings.py index dc01d96f99e..f38bc170352 100644 --- a/services/storage/src/simcore_service_storage/settings.py +++ b/services/storage/src/simcore_service_storage/settings.py @@ -1,6 +1,13 @@ -from typing import Any - -from pydantic import Field, PositiveInt, root_validator, validator +from typing import Self + +from pydantic import ( + AliasChoices, + Field, + PositiveInt, + TypeAdapter, + field_validator, + model_validator, +) from settings_library.base import BaseCustomSettings from settings_library.basic_types import LogLevel, PortInt from settings_library.postgres import PostgresSettings @@ -14,10 +21,11 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): STORAGE_HOST: str = "0.0.0.0" # nosec - STORAGE_PORT: PortInt = PortInt(8080) + STORAGE_PORT: PortInt = TypeAdapter(PortInt).validate_python(8080) LOG_LEVEL: LogLevel = Field( - "INFO", env=["STORAGE_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + "INFO", + validation_alias=AliasChoices("STORAGE_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ) STORAGE_MAX_WORKERS: PositiveInt = Field( @@ -35,15 +43,23 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): None, description="Pennsieve API secret ONLY for testing purposes" ) - STORAGE_POSTGRES: PostgresSettings = Field(auto_default_from_env=True) + STORAGE_POSTGRES: PostgresSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) - STORAGE_REDIS: RedisSettings | None = Field(auto_default_from_env=True) + STORAGE_REDIS: RedisSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - STORAGE_S3: S3Settings = Field(auto_default_from_env=True) + STORAGE_S3: S3Settings = Field(json_schema_extra={"auto_default_from_env": True}) - STORAGE_TRACING: TracingSettings | None = Field(auto_default_from_env=True) + STORAGE_TRACING: TracingSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) - DATCORE_ADAPTER: DatcoreAdapterSettings = Field(auto_default_from_env=True) + DATCORE_ADAPTER: DatcoreAdapterSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) STORAGE_SYNC_METADATA_TIMEOUT: PositiveInt = Field( 180, description="Timeout (seconds) for metadata sync task" @@ -64,22 +80,26 @@ class Settings(BaseCustomSettings, MixinLoggingSettings): ) STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - False, - env=["STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"], + default=False, + validation_alias=AliasChoices( + "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) - @validator("LOG_LEVEL") + @field_validator("LOG_LEVEL", mode="before") @classmethod - def _validate_loglevel(cls, value) -> str: + def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) return log_level - @root_validator() - @classmethod - def ensure_settings_consistency(cls, values: dict[str, Any]): - if values.get("STORAGE_CLEANER_INTERVAL_S") and not values.get("STORAGE_REDIS"): - raise ValueError( - "STORAGE_CLEANER_INTERVAL_S cleaner cannot be set without STORAGE_REDIS! Please correct settings." + @model_validator(mode="after") + def ensure_settings_consistency(self) -> Self: + if self.STORAGE_CLEANER_INTERVAL_S is not None and not self.STORAGE_REDIS: + msg = ( + "STORAGE_CLEANER_INTERVAL_S cleaner cannot be set without STORAGE_REDIS! " + "Please correct settings." ) - return values + raise ValueError(msg) + return self diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index db5a1ab288b..b6e7e57f1cc 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -20,7 +20,12 @@ S3MetaData, UploadedBytesTransferredCallback, ) -from models_library.api_schemas_storage import LinkType, S3BucketName, UploadedPart +from models_library.api_schemas_storage import ( + UNDEFINED_SIZE_TYPE, + LinkType, + S3BucketName, + UploadedPart, +) from models_library.basic_types import SHA256Str from models_library.projects import ProjectID from models_library.projects_nodes_io import ( @@ -30,7 +35,7 @@ StorageFileID, ) from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, NonNegativeInt, parse_obj_as +from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter from servicelib.aiohttp.client_session import get_client_session from servicelib.aiohttp.long_running_tasks.server import TaskProgress from servicelib.logging_utils import log_context @@ -252,7 +257,7 @@ async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaDat raise FileAccessRightError(access_right="read", file_id=file_id) fmd = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if is_file_entry_valid(fmd): return convert_db_to_model(fmd) @@ -279,7 +284,7 @@ async def create_file_upload_links( # there was a multipart upload in progress beforehand, it MUST be # cancelled to prevent unwanted costs in AWS await self._clean_pending_upload( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if ( @@ -352,7 +357,7 @@ async def create_file_upload_links( # user wants just the s3 link s3_link = get_s3_client(self.app).compute_s3_url( bucket=self.simcore_bucket_name, - object_key=parse_obj_as(SimcoreS3FileID, file_id), + object_key=TypeAdapter(SimcoreS3FileID).validate_python(file_id), ) return UploadLinks( [s3_link], file_size_bytes or MAX_LINK_CHUNK_BYTE_SIZE[link_type] @@ -371,7 +376,7 @@ async def abort_file_upload( raise FileAccessRightError(access_right="write/delete", file_id=file_id) fmd: FileMetaDataAtDB = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if is_valid_managed_multipart_upload(fmd.upload_id): assert fmd.upload_id # nosec @@ -407,7 +412,7 @@ async def complete_file_upload( if not can.write: raise FileAccessRightError(access_right="write", file_id=file_id) fmd = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if is_valid_managed_multipart_upload(fmd.upload_id): @@ -455,12 +460,12 @@ async def create_file_download_link( ): raise S3KeyNotFoundError(key=file_id, bucket=self.simcore_bucket_name) return await self.__get_link( - parse_obj_as(SimcoreS3FileID, file_id), link_type + TypeAdapter(SimcoreS3FileID).validate_python(file_id), link_type ) # standard file link async with self.engine.acquire() as conn: fmd = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) if not is_file_entry_valid(fmd): # try lazy update @@ -482,9 +487,8 @@ async def __ensure_read_access_rights( async def __get_link( self, s3_file_id: SimcoreS3FileID, link_type: LinkType ) -> AnyUrl: - link: AnyUrl = parse_obj_as( - AnyUrl, - f"s3://{self.simcore_bucket_name}/{urllib.parse.quote(s3_file_id)}", + link: AnyUrl = TypeAdapter(AnyUrl).validate_python( + f"s3://{self.simcore_bucket_name}/{urllib.parse.quote(s3_file_id)}" ) if link_type == LinkType.PRESIGNED: link = await get_s3_client(self.app).create_single_presigned_download_link( @@ -523,7 +527,7 @@ async def delete_file( # NOTE: deleting might be slow, so better ensure we release the connection async with self.engine.acquire() as conn: file: FileMetaDataAtDB = await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(file_id) ) await get_s3_client(self.app).delete_objects_recursively( bucket=file.bucket_name, @@ -620,14 +624,16 @@ async def deep_copy_project_simcore_s3( f"{len(src_project_files)} files", log_duration=True, ): - sizes_and_num_files: list[tuple[ByteSize, int]] = await limited_gather( + sizes_and_num_files: list[ + tuple[ByteSize | UNDEFINED_SIZE_TYPE, int] + ] = await limited_gather( *[self._get_size_and_num_files(fmd) for fmd in src_project_files], limit=_MAX_PARALLEL_S3_CALLS, ) total_num_of_files = sum(n for _, n in sizes_and_num_files) - src_project_total_data_size: ByteSize = parse_obj_as( - ByteSize, sum(n for n, _ in sizes_and_num_files) - ) + src_project_total_data_size: ByteSize = TypeAdapter( + ByteSize + ).validate_python(sum(n for n, _ in sizes_and_num_files)) with log_context( _logger, logging.INFO, @@ -653,7 +659,7 @@ async def deep_copy_project_simcore_s3( self._copy_path_s3_s3( user_id, src_fmd=src_fmd, - dst_file_id=SimcoreS3FileID( + dst_file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{dst_project_uuid}/{new_node_id}/{src_fmd.object_name.split('/', maxsplit=2)[-1]}" ), bytes_transfered_cb=s3_transfered_data_cb.copy_transfer_cb, @@ -693,7 +699,7 @@ async def deep_copy_project_simcore_s3( async def _get_size_and_num_files( self, fmd: FileMetaDataAtDB - ) -> tuple[ByteSize, int]: + ) -> tuple[ByteSize | UNDEFINED_SIZE_TYPE, int]: if not fmd.is_directory: return fmd.file_size, 1 @@ -711,7 +717,7 @@ async def _get_size_and_num_files( total_size += sum(x.size for x in s3_objects) total_num_s3_objects += len(s3_objects) - return parse_obj_as(ByteSize, total_size), total_num_s3_objects + return TypeAdapter(ByteSize).validate_python(total_size), total_num_s3_objects async def search_owned_files( self, @@ -752,7 +758,7 @@ async def create_soft_link( ) -> FileMetaData: async with self.engine.acquire() as conn: if await db_file_meta_data.exists( - conn, parse_obj_as(SimcoreS3FileID, link_file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(link_file_id) ): raise LinkAlreadyExistsError(file_id=link_file_id) # validate target_uuid @@ -901,7 +907,7 @@ async def _update_fmd_from_other( s3_metadata = await get_s3_client(self.app).get_object_metadata( bucket=fmd.bucket_name, object_key=fmd.object_name ) - fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size) + fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.last_modified = s3_metadata.last_modified fmd.entity_tag = s3_metadata.e_tag else: @@ -938,12 +944,12 @@ async def _update_database_from_storage( s3_metadata = await self._get_s3_metadata(fmd) if not fmd.is_directory: assert isinstance(s3_metadata, S3MetaData) # nosec - fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size) + fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.last_modified = s3_metadata.last_modified fmd.entity_tag = s3_metadata.e_tag elif fmd.is_directory: assert isinstance(s3_metadata, S3DirectoryMetaData) # nosec - fmd.file_size = parse_obj_as(ByteSize, s3_metadata.size) + fmd.file_size = TypeAdapter(ByteSize).validate_python(s3_metadata.size) fmd.upload_expires_at = None fmd.upload_id = None async with self.engine.acquire() as conn: @@ -971,13 +977,15 @@ async def _copy_file_datcore_s3( ) assert dc_link.path # nosec filename = Path(dc_link.path).name - dst_file_id = SimcoreS3FileID(f"{dest_project_id}/{dest_node_id}/{filename}") + dst_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{dest_project_id}/{dest_node_id}/{filename}" + ) _logger.debug("copying %s to %s", f"{source_uuid=}", f"{dst_file_id=}") with tempfile.TemporaryDirectory() as tmpdir: local_file_path = Path(tmpdir) / filename # Downloads DATCore -> local - await download_to_file_or_raise(session, dc_link, local_file_path) + await download_to_file_or_raise(session, f"{dc_link}", local_file_path) # copying will happen using aioboto3, therefore multipart might happen async with self.engine.acquire() as conn: @@ -1068,7 +1076,7 @@ async def _create_fmd_for_upload( ) fmd = FileMetaData.from_simcore_node( user_id=user_id, - file_id=parse_obj_as(SimcoreS3FileID, file_id), + file_id=TypeAdapter(SimcoreS3FileID).validate_python(file_id), bucket=self.simcore_bucket_name, location_id=self.location_id, location_name=self.location_name, @@ -1085,7 +1093,9 @@ def create_simcore_s3_data_manager(app: web.Application) -> SimcoreS3DataManager assert cfg.STORAGE_S3 # nosec return SimcoreS3DataManager( engine=app[APP_AIOPG_ENGINE_KEY], - simcore_bucket_name=parse_obj_as(S3BucketName, cfg.STORAGE_S3.S3_BUCKET_NAME), + simcore_bucket_name=TypeAdapter(S3BucketName).validate_python( + cfg.STORAGE_S3.S3_BUCKET_NAME + ), app=app, settings=cfg, ) diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py index 3cb3cbfc399..e4a58549e31 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py @@ -1,6 +1,5 @@ from contextlib import suppress from pathlib import Path -from typing import cast from aiopg.sa.connection import SAConnection from aws_library.s3 import S3MetaData, SimcoreS3API @@ -10,7 +9,7 @@ SimcoreS3FileID, StorageFileID, ) -from pydantic import ByteSize, NonNegativeInt, parse_obj_as +from pydantic import ByteSize, NonNegativeInt, TypeAdapter from servicelib.utils import ensure_ends_with from . import db_file_meta_data @@ -56,7 +55,7 @@ async def expand_directory( location_id=fmd.location_id, location=fmd.location, bucket_name=fmd.bucket_name, - object_name=cast(SimcoreS3FileID, x.object_key), + object_name=x.object_key, user_id=fmd.user_id, # NOTE: to ensure users have a consistent experience the # `created_at` field is inherited from the last_modified @@ -64,8 +63,8 @@ async def expand_directory( # creation of the directory, the file's creation date # will not be 1 month in the passed. created_at=x.last_modified, - file_id=cast(SimcoreS3FileID, x.object_key), - file_size=parse_obj_as(ByteSize, x.size), + file_id=x.object_key, + file_size=TypeAdapter(ByteSize).validate_python(x.size), last_modified=x.last_modified, entity_tag=x.e_tag, is_soft_link=False, @@ -100,7 +99,7 @@ async def _get_fmd( ) -> FileMetaDataAtDB | None: with suppress(FileMetaDataNotFoundError): return await db_file_meta_data.get( - conn, parse_obj_as(SimcoreS3FileID, s3_file_id) + conn, TypeAdapter(SimcoreS3FileID).validate_python(s3_file_id) ) return None @@ -114,7 +113,9 @@ async def _get_fmd( # could not extract a directory name from the provided path return None - directory_file_id = parse_obj_as(SimcoreS3FileID, directory_file_id_str) + directory_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + directory_file_id_str + ) directory_file_id_fmd = await _get_fmd(conn, directory_file_id) return directory_file_id if directory_file_id_fmd else None diff --git a/services/storage/src/simcore_service_storage/utils.py b/services/storage/src/simcore_service_storage/utils.py index 0baddfcfc9a..7abc18ed552 100644 --- a/services/storage/src/simcore_service_storage/utils.py +++ b/services/storage/src/simcore_service_storage/utils.py @@ -15,8 +15,8 @@ def convert_db_to_model(x: FileMetaDataAtDB) -> FileMetaData: - model: FileMetaData = FileMetaData.parse_obj( - x.dict() + model: FileMetaData = FileMetaData.model_validate( + x.model_dump() | { "file_uuid": x.file_id, "file_name": x.file_id.split("/")[-1], diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 1b3f634446c..ca6483c75ae 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -39,7 +39,7 @@ from models_library.projects_nodes_io import LocationID, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context @@ -174,15 +174,15 @@ def app_settings( monkeypatch.delenv("S3_ENDPOINT") setenvs_from_dict( monkeypatch, - s3_settings.dict(exclude={"S3_ENDPOINT"}), + s3_settings.model_dump(exclude={"S3_ENDPOINT"}), ) else: setenvs_from_dict( monkeypatch, - s3_settings.dict(), + s3_settings.model_dump(), ) test_app_settings = Settings.create_from_envs() - print(f"{test_app_settings.json(indent=2)=}") + print(f"{test_app_settings.model_dump_json(indent=2)=}") return test_app_settings @@ -259,7 +259,7 @@ async def _getter(file_id: SimcoreS3FileID) -> FileMetaDataGet: data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_fmd = parse_obj_as(FileMetaDataGet, data) + received_fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert received_fmd return received_fmd @@ -291,7 +291,7 @@ async def _link_creator( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_file_upload = parse_obj_as(FileUploadSchema, data) + received_file_upload = TypeAdapter(FileUploadSchema).validate_python(data) assert received_file_upload file_params.append((user_id, location_id, file_id)) return received_file_upload @@ -353,7 +353,7 @@ async def _uploader( file, file_upload_link ) # complete the upload - complete_url = URL(file_upload_link.links.complete_upload).relative() + complete_url = URL(f"{file_upload_link.links.complete_upload}").relative() with log_context(logging.INFO, f"completing upload of {file=}"): response = await client.post( f"{complete_url}", @@ -363,8 +363,10 @@ async def _uploader( data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate( + data + ) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() completion_etag = None async for attempt in AsyncRetrying( @@ -382,7 +384,7 @@ async def _uploader( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) if future.state == FileUploadCompleteState.NOK: msg = f"{data=}" raise ValueError(msg) @@ -430,7 +432,7 @@ def _creator( if file_base_path: s3_file_name = f"{file_base_path / file_name}" clean_path = Path(f"{project_id}/{node_id}/{s3_file_name}") - return SimcoreS3FileID(f"{clean_path}") + return TypeAdapter(SimcoreS3FileID).validate_python(f"{clean_path}") return _creator @@ -470,7 +472,7 @@ async def _directory_creator(dir_name: str): assert len(directory_file_upload.urls) == 1 # complete the upload - complete_url = URL(directory_file_upload.links.complete_upload).relative() + complete_url = URL(f"{directory_file_upload.links.complete_upload}").relative() response = await client.post( f"{complete_url}", json=jsonable_encoder(FileUploadCompletionBody(parts=[])), @@ -479,8 +481,8 @@ async def _directory_creator(dir_name: str): data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() # check that it finished updating assert client.app @@ -500,7 +502,7 @@ async def _directory_creator(dir_name: str): data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) assert future.state == FileUploadCompleteState.OK assert future.e_tag is None ctx.logger.info( @@ -535,7 +537,9 @@ async def _create_file(s: int, f: int): await storage_s3_client.upload_file( bucket=storage_s3_bucket, file=file, - object_key=SimcoreS3FileID(f"{clean_path}"), + object_key=TypeAdapter(SimcoreS3FileID).validate_python( + f"{clean_path}" + ), bytes_transfered_cb=None, ) diff --git a/services/storage/tests/fixtures/data_models.py b/services/storage/tests/fixtures/data_models.py index ae5816a427f..ab225928f62 100644 --- a/services/storage/tests/fixtures/data_models.py +++ b/services/storage/tests/fixtures/data_models.py @@ -18,7 +18,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.faker_factories import random_project, random_user from servicelib.utils import limited_gather from simcore_postgres_database.models.project_to_groups import project_to_groups @@ -45,7 +45,7 @@ async def _user_context(aiopg_engine: Engine, *, name: str) -> AsyncIterator[Use assert isinstance(row.id, int) try: - yield UserID(row.id) + yield TypeAdapter(UserID).validate_python(row.id) finally: async with aiopg_engine.acquire() as conn: await conn.execute(users.delete().where(users.c.id == row.id)) @@ -149,7 +149,7 @@ async def project_id( async def collaborator_id(aiopg_engine: Engine) -> AsyncIterator[UserID]: async with _user_context(aiopg_engine, name="collaborator") as new_user_id: - yield UserID(new_user_id) + yield TypeAdapter(UserID).validate_python(new_user_id) @pytest.fixture @@ -177,7 +177,7 @@ async def _() -> None: ) row = await result.fetchone() assert row - access_rights: dict[str, Any] = row[projects.c.access_rights] + access_rights: dict[str | int, Any] = row[projects.c.access_rights] access_rights[await _get_user_group(conn, user_id)] = { "read": True, @@ -279,22 +279,19 @@ async def random_project_with_files( async def _creator( num_nodes: int = 12, file_sizes: tuple[ByteSize, ...] = ( - parse_obj_as(ByteSize, "7Mib"), - parse_obj_as(ByteSize, "110Mib"), - parse_obj_as(ByteSize, "1Mib"), + TypeAdapter(ByteSize).validate_python("7Mib"), + TypeAdapter(ByteSize).validate_python("110Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), ), file_checksums: tuple[SHA256Str, ...] = ( - parse_obj_as( - SHA256Str, - "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd", + TypeAdapter(SHA256Str).validate_python( + "311e2e130d83cfea9c3b7560699c221b0b7f9e5d58b02870bd52b695d8b4aabd" ), - parse_obj_as( - SHA256Str, - "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e", + TypeAdapter(SHA256Str).validate_python( + "08e297db979d3c84f6b072c2a1e269e8aa04e82714ca7b295933a0c9c0f62b2e" ), - parse_obj_as( - SHA256Str, - "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3", + TypeAdapter(SHA256Str).validate_python( + "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3" ), ), ) -> tuple[ diff --git a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py index 39be9386497..5e94a17d3bc 100644 --- a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py +++ b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py @@ -56,7 +56,7 @@ def location_name() -> str: return SimcoreS3DataManager.get_location_name() -async def test_storage_client_used_in_simcore_sdk_0_3_2( +async def test_storage_client_used_in_simcore_sdk_0_3_2( # noqa: PLR0915 client: TestClient, file_id: str, user_id: str, diff --git a/services/storage/tests/unit/test_cli.py b/services/storage/tests/unit/test_cli.py index cab69609fdd..7c86b85fcd7 100644 --- a/services/storage/tests/unit/test_cli.py +++ b/services/storage/tests/unit/test_cli.py @@ -29,7 +29,7 @@ def test_cli_settings_as_json( assert result.exit_code == os.EX_OK, result # reuse resulting json to build settings settings: dict = json.loads(result.stdout) - assert Settings.parse_obj(settings) + assert Settings.model_validate(settings) def test_cli_settings_env_file( @@ -41,9 +41,9 @@ def test_cli_settings_env_file( # reuse resulting env_file to build settings env_file = StringIO(result.stdout) - settings: dict = dotenv_values(stream=env_file) + settings = dotenv_values(stream=env_file) for key, value in settings.items(): with contextlib.suppress(json.decoder.JSONDecodeError): settings[key] = json.loads(str(value)) - assert Settings.parse_obj(settings) + assert Settings.model_validate(settings) diff --git a/services/storage/tests/unit/test_dsm.py b/services/storage/tests/unit/test_dsm.py index ae07ec94c9b..22c78955581 100644 --- a/services/storage/tests/unit/test_dsm.py +++ b/services/storage/tests/unit/test_dsm.py @@ -10,7 +10,7 @@ from faker import Faker from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from servicelib.utils import limited_gather from simcore_service_storage.models import FileMetaData, S3BucketName from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -31,7 +31,7 @@ async def dsm_mockup_complete_db( cleanup_user_projects_file_metadata: None, faker: Faker, ) -> tuple[FileMetaData, FileMetaData]: - file_size = parse_obj_as(ByteSize, "10Mib") + file_size = TypeAdapter(ByteSize).validate_python("10Mib") uploaded_files = await limited_gather( *(upload_file(file_size, faker.file_name(), None) for _ in range(2)), limit=2, diff --git a/services/storage/tests/unit/test_dsm_dsmcleaner.py b/services/storage/tests/unit/test_dsm_dsmcleaner.py index 60f4f7f57a9..7862d4d7166 100644 --- a/services/storage/tests/unit/test_dsm_dsmcleaner.py +++ b/services/storage/tests/unit/test_dsm_dsmcleaner.py @@ -3,7 +3,6 @@ # pylint: disable=redefined-outer-name # pylint: disable=too-many-arguments # pylint: disable=too-many-branches -# pylint: disable=too-many-positional-arguments # pylint: disable=unused-argument # pylint: disable=unused-variable @@ -23,7 +22,7 @@ from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import SimcoreS3DirectoryID, SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.parametrizations import byte_size_ids from simcore_postgres_database.storage_models import file_meta_data from simcore_service_storage import db_file_meta_data @@ -47,14 +46,18 @@ def disabled_dsm_cleaner_task(monkeypatch: pytest.MonkeyPatch): @pytest.fixture def simcore_directory_id(simcore_file_id: SimcoreS3FileID) -> SimcoreS3FileID: - return SimcoreS3FileID( - Path(SimcoreS3DirectoryID.from_simcore_s3_object(simcore_file_id)) + return TypeAdapter(SimcoreS3FileID).validate_python( + SimcoreS3DirectoryID.from_simcore_s3_object(simcore_file_id) ) @pytest.mark.parametrize( "file_size", - [ByteSize(0), parse_obj_as(ByteSize, "10Mib"), parse_obj_as(ByteSize, "100Mib")], + [ + TypeAdapter(ByteSize).validate_python("0"), + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), + ], ids=byte_size_ids, ) @pytest.mark.parametrize( @@ -120,7 +123,11 @@ async def test_regression_collaborator_creates_file_upload_links( @pytest.mark.parametrize( "file_size", - [ByteSize(0), parse_obj_as(ByteSize, "10Mib"), parse_obj_as(ByteSize, "100Mib")], + [ + ByteSize(0), + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), + ], ids=byte_size_ids, ) @pytest.mark.parametrize( @@ -207,7 +214,10 @@ async def test_clean_expired_uploads_deletes_expired_pending_uploads( @pytest.mark.parametrize( "file_size", - [parse_obj_as(ByteSize, "10Mib"), parse_obj_as(ByteSize, "100Mib")], + [ + TypeAdapter(ByteSize).validate_python("10Mib"), + TypeAdapter(ByteSize).validate_python("100Mib"), + ], ids=byte_size_ids, ) @pytest.mark.parametrize("link_type", [LinkType.S3, LinkType.PRESIGNED]) @@ -287,7 +297,7 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi # check the entries were reverted async with aiopg_engine.acquire() as conn: reverted_fmd = await db_file_meta_data.get(conn, file_id) - assert original_fmd.dict(exclude={"created_at"}) == reverted_fmd.dict( + assert original_fmd.model_dump(exclude={"created_at"}) == reverted_fmd.model_dump( exclude={"created_at"} ) # check the S3 content is the old file @@ -303,7 +313,7 @@ async def test_clean_expired_uploads_reverts_to_last_known_version_expired_pendi @pytest.mark.parametrize( "file_size", - [parse_obj_as(ByteSize, "100Mib")], + [TypeAdapter(ByteSize).validate_python("100Mib")], ids=byte_size_ids, ) @pytest.mark.parametrize("is_directory", [True, False]) @@ -353,7 +363,9 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation file_ids_to_upload: set[SimcoreS3FileID] = ( { - SimcoreS3FileID(f"{file_or_directory_id}/file{x}") + TypeAdapter(SimcoreS3FileID).validate_python( + f"{file_or_directory_id}/file{x}" + ) for x in range(FILES_IN_DIR) } if is_directory @@ -366,7 +378,7 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation object_key=file_id, file_size=file_size, expiration_secs=3600, - sha256_checksum=parse_obj_as(SHA256Str, _faker.sha256()), + sha256_checksum=TypeAdapter(SHA256Str).validate_python(_faker.sha256()), ) for file_id in file_ids_to_upload ] diff --git a/services/storage/tests/unit/test_dsm_soft_links.py b/services/storage/tests/unit/test_dsm_soft_links.py index ed0e01ea7f0..dd822ea2165 100644 --- a/services/storage/tests/unit/test_dsm_soft_links.py +++ b/services/storage/tests/unit/test_dsm_soft_links.py @@ -46,7 +46,7 @@ async def output_file( async with aiopg_engine.acquire() as conn: stmt = ( file_meta_data.insert() - .values(jsonable_encoder(FileMetaDataAtDB.from_orm(file))) + .values(jsonable_encoder(FileMetaDataAtDB.model_validate(file))) .returning(literal_column("*")) ) result = await conn.execute(stmt) diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 92408a33136..d207005c2d2 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -15,7 +15,7 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.parametrizations import ( @@ -73,7 +73,7 @@ async def test_get_files_metadata_dataset( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (n + 1) fmd = list_fmds[n] assert fmd.file_name == file.name @@ -100,7 +100,7 @@ async def test_get_datasets_metadata( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - list_datasets = parse_obj_as(list[DatasetMetaDataGet], data) + list_datasets = TypeAdapter(list[DatasetMetaDataGet]).validate_python(data) assert len(list_datasets) == 1 dataset = list_datasets[0] assert dataset.dataset_id == project_id diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index 5623c7e67c2..f9fc415d86a 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -11,12 +11,12 @@ import json import logging import urllib.parse -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import AbstractAsyncContextManager from dataclasses import dataclass from pathlib import Path from random import choice -from typing import Any, AsyncIterator, Literal +from typing import Any, Literal from uuid import uuid4 import pytest @@ -43,7 +43,7 @@ from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import AnyHttpUrl, ByteSize, HttpUrl, parse_obj_as +from pydantic import AnyHttpUrl, ByteSize, HttpUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context @@ -112,7 +112,7 @@ class SingleLinkParam: {}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="default_returns_single_presigned", ), @@ -121,13 +121,16 @@ class SingleLinkParam: {"link_type": "presigned"}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="presigned_returns_single_presigned", ), pytest.param( SingleLinkParam( - {"link_type": "s3"}, "s3", [], parse_obj_as(ByteSize, "5TiB") + {"link_type": "s3"}, + "s3", + [], + TypeAdapter(ByteSize).validate_python("5TiB"), ), id="s3_returns_single_s3_link", ), @@ -207,7 +210,7 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_file_upload_link = parse_obj_as(PresignedLink, data) + received_file_upload_link = TypeAdapter(PresignedLink).validate_python(data) assert received_file_upload_link file_params.append((user_id, location_id, file_id)) return received_file_upload_link @@ -238,7 +241,7 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi {}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="default_returns_single_presigned", ), @@ -247,13 +250,16 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi {"link_type": "presigned"}, "http", _HTTP_PRESIGNED_LINK_QUERY_KEYS, - parse_obj_as(ByteSize, "5GiB"), + TypeAdapter(ByteSize).validate_python("5GiB"), ), id="presigned_returns_single_presigned", ), pytest.param( SingleLinkParam( - {"link_type": "s3"}, "s3", [], parse_obj_as(ByteSize, "5TiB") + {"link_type": "s3"}, + "s3", + [], + TypeAdapter(ByteSize).validate_python("5TiB"), ), id="s3_returns_single_s3_link", ), @@ -314,50 +320,50 @@ class MultiPartParam: pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "10MiB"), + file_size=TypeAdapter(ByteSize).validate_python("10MiB"), expected_response=status.HTTP_200_OK, expected_num_links=1, - expected_chunk_size=parse_obj_as(ByteSize, "10MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("10MiB"), ), id="10MiB file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "100MiB"), + file_size=TypeAdapter(ByteSize).validate_python("100MiB"), expected_response=status.HTTP_200_OK, expected_num_links=10, - expected_chunk_size=parse_obj_as(ByteSize, "10MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("10MiB"), ), id="100MiB file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "5TiB"), + file_size=TypeAdapter(ByteSize).validate_python("5TiB"), expected_response=status.HTTP_200_OK, expected_num_links=8739, - expected_chunk_size=parse_obj_as(ByteSize, "600MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("600MiB"), ), id="5TiB file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.PRESIGNED, - file_size=parse_obj_as(ByteSize, "9431773844"), + file_size=TypeAdapter(ByteSize).validate_python("9431773844"), expected_response=status.HTTP_200_OK, expected_num_links=900, - expected_chunk_size=parse_obj_as(ByteSize, "10MiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("10MiB"), ), id="9431773844B (8.8Gib) file,presigned", ), pytest.param( MultiPartParam( link_type=LinkType.S3, - file_size=parse_obj_as(ByteSize, "255GiB"), + file_size=TypeAdapter(ByteSize).validate_python("255GiB"), expected_response=status.HTTP_200_OK, expected_num_links=1, - expected_chunk_size=parse_obj_as(ByteSize, "255GiB"), + expected_chunk_size=TypeAdapter(ByteSize).validate_python("255GiB"), ), id="5TiB file,s3", ), @@ -409,8 +415,8 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin @pytest.mark.parametrize( "link_type, file_size", [ - (LinkType.PRESIGNED, parse_obj_as(ByteSize, "1000Mib")), - (LinkType.S3, parse_obj_as(ByteSize, "1000Mib")), + (LinkType.PRESIGNED, TypeAdapter(ByteSize).validate_python("1000Mib")), + (LinkType.S3, TypeAdapter(ByteSize).validate_python("1000Mib")), ], ids=byte_size_ids, ) @@ -449,7 +455,7 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( expected_upload_ids=([upload_id] if upload_id else None), ) # delete/abort file upload - abort_url = URL(upload_link.links.abort_upload).relative() + abort_url = URL(f"{upload_link.links.abort_upload}").relative() response = await client.post(f"{abort_url}") await assert_status(response, status.HTTP_204_NO_CONTENT) @@ -474,10 +480,10 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( @pytest.mark.parametrize( "link_type, file_size", [ - (LinkType.PRESIGNED, parse_obj_as(ByteSize, "10Mib")), - (LinkType.PRESIGNED, parse_obj_as(ByteSize, "1000Mib")), - (LinkType.S3, parse_obj_as(ByteSize, "10Mib")), - (LinkType.S3, parse_obj_as(ByteSize, "1000Mib")), + (LinkType.PRESIGNED, TypeAdapter(ByteSize).validate_python("10Mib")), + (LinkType.PRESIGNED, TypeAdapter(ByteSize).validate_python("1000Mib")), + (LinkType.S3, TypeAdapter(ByteSize).validate_python("10Mib")), + (LinkType.S3, TypeAdapter(ByteSize).validate_python("1000Mib")), ], ids=byte_size_ids, ) @@ -561,9 +567,11 @@ def complex_file_name(faker: Faker) -> str: @pytest.mark.parametrize( "file_size", [ - (parse_obj_as(ByteSize, "1Mib")), - (parse_obj_as(ByteSize, "500Mib")), - pytest.param(parse_obj_as(ByteSize, "7Gib"), marks=pytest.mark.heavy_load), + (TypeAdapter(ByteSize).validate_python("1Mib")), + (TypeAdapter(ByteSize).validate_python("500Mib")), + pytest.param( + TypeAdapter(ByteSize).validate_python("5Gib"), marks=pytest.mark.heavy_load + ), ], ids=byte_size_ids, ) @@ -578,8 +586,8 @@ async def test_upload_real_file( @pytest.mark.parametrize( "file_size", [ - (parse_obj_as(ByteSize, "1Mib")), - (parse_obj_as(ByteSize, "117Mib")), + (TypeAdapter(ByteSize).validate_python("1Mib")), + (TypeAdapter(ByteSize).validate_python("117Mib")), ], ids=byte_size_ids, ) @@ -614,7 +622,7 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w file, file_upload_link ) # complete the upload - complete_url = URL(file_upload_link.links.complete_upload).relative() + complete_url = URL(f"{file_upload_link.links.complete_upload}").relative() response = await client.post( f"{complete_url}", json=jsonable_encoder(FileUploadCompletionBody(parts=part_to_etag)), @@ -623,8 +631,8 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() # here we do not check now for the state completion. instead we simulate a restart where the tasks disappear client.app[UPLOAD_TASKS_KEY].clear() @@ -644,7 +652,7 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) assert future.state == FileUploadCompleteState.OK assert future.e_tag is not None completion_etag = future.e_tag @@ -686,7 +694,7 @@ async def test_upload_of_single_presigned_link_lazily_update_database_on_get( s3_client: S3Client, ): assert client.app - file_size = parse_obj_as(ByteSize, "500Mib") + file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() # create a file file = create_file_of_size(file_size, file_name) @@ -729,7 +737,7 @@ async def test_upload_real_file_with_s3_client( s3_client: S3Client, ): assert client.app - file_size = parse_obj_as(ByteSize, "500Mib") + file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() # create a file file = create_file_of_size(file_size, file_name) @@ -754,15 +762,15 @@ async def test_upload_real_file_with_s3_client( assert s3_metadata.e_tag == upload_e_tag # complete the upload - complete_url = URL(file_upload_link.links.complete_upload).relative() + complete_url = URL(f"{file_upload_link.links.complete_upload}").relative() with log_context(logging.INFO, f"completing upload of {file=}"): response = await client.post(f"{complete_url}", json={"parts": []}) response.raise_for_status() data, error = await assert_status(response, status.HTTP_202_ACCEPTED) assert not error assert data - file_upload_complete_response = FileUploadCompleteResponse.parse_obj(data) - state_url = URL(file_upload_complete_response.links.state).relative() + file_upload_complete_response = FileUploadCompleteResponse.model_validate(data) + state_url = URL(f"{file_upload_complete_response.links.state}").relative() completion_etag = None async for attempt in AsyncRetrying( reraise=True, @@ -779,7 +787,7 @@ async def test_upload_real_file_with_s3_client( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - future = FileUploadCompleteFutureResponse.parse_obj(data) + future = FileUploadCompleteFutureResponse.model_validate(data) if future.state != FileUploadCompleteState.OK: msg = f"{data=}" raise ValueError(msg) @@ -812,7 +820,10 @@ async def test_upload_real_file_with_s3_client( @pytest.mark.parametrize( "file_size", - [parse_obj_as(ByteSize, "160Mib"), parse_obj_as(ByteSize, "1Mib")], + [ + TypeAdapter(ByteSize).validate_python("160Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + ], ids=byte_size_ids, ) async def test_upload_twice_and_fail_second_time_shall_keep_first_version( @@ -865,7 +876,7 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( ) # 4. abort file upload - abort_url = URL(upload_link.links.abort_upload).relative() + abort_url = URL(f"{upload_link.links.abort_upload}").relative() response = await client.post(f"{abort_url}") await assert_status(response, status.HTTP_204_NO_CONTENT) @@ -888,7 +899,7 @@ async def test_upload_twice_and_fail_second_time_shall_keep_first_version( @pytest.fixture def file_size() -> ByteSize: - return parse_obj_as(ByteSize, "1Mib") + return TypeAdapter(ByteSize).validate_python("1Mib") async def _assert_file_downloaded( @@ -896,7 +907,7 @@ async def _assert_file_downloaded( ): dest_file = tmp_path / faker.file_name() async with ClientSession() as session: - response = await session.get(link) + response = await session.get(f"{link}") response.raise_for_status() with dest_file.open("wb") as fp: fp.write(await response.read()) @@ -916,7 +927,9 @@ async def test_download_file_no_file_was_uploaded( ): assert client.app - missing_file = parse_obj_as(SimcoreS3FileID, f"{project_id}/{node_id}/missing.file") + missing_file = TypeAdapter(SimcoreS3FileID).validate_python( + f"{project_id}/{node_id}/missing.file" + ) assert ( await storage_s3_client.object_exists( bucket=storage_s3_bucket, object_key=missing_file @@ -975,7 +988,7 @@ async def test_download_file_1_to_1_with_file_meta_data( assert not error assert data assert "link" in data - assert parse_obj_as(AnyHttpUrl, data["link"]) + assert TypeAdapter(AnyHttpUrl).validate_python(data["link"]) await _assert_file_downloaded( faker, tmp_path, link=data["link"], uploaded_file=uploaded_file ) @@ -1006,7 +1019,9 @@ async def test_download_file_from_inside_a_directory( file_name = "meta_data_entry_is_dir.file" file_to_upload_in_dir = create_file_of_size(file_size, file_name) - s3_file_id = parse_obj_as(SimcoreS3FileID, f"{dir_path_in_s3}/{file_name}") + s3_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{dir_path_in_s3}/{file_name}" + ) await storage_s3_client.upload_file( bucket=storage_s3_bucket, file=file_to_upload_in_dir, @@ -1034,7 +1049,7 @@ async def test_download_file_from_inside_a_directory( assert not error assert data assert "link" in data - assert parse_obj_as(AnyHttpUrl, data["link"]) + assert TypeAdapter(AnyHttpUrl).validate_python(data["link"]) await _assert_file_downloaded( faker, tmp_path, link=data["link"], uploaded_file=file_to_upload_in_dir ) @@ -1055,8 +1070,8 @@ async def test_download_file_the_file_is_missing_from_the_directory( assert directory_file_upload.urls[0].path dir_path_in_s3 = directory_file_upload.urls[0].path.strip("/") - missing_s3_file_id = parse_obj_as( - SimcoreS3FileID, f"{dir_path_in_s3}/missing_inside_dir.file" + missing_s3_file_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"{dir_path_in_s3}/missing_inside_dir.file" ) download_url = ( client.app.router["download_file"] @@ -1083,8 +1098,8 @@ async def test_download_file_access_rights( assert client.app # project_id does not exist - missing_file = parse_obj_as( - SimcoreS3FileID, f"{faker.uuid4()}/{faker.uuid4()}/project_id_is_missing" + missing_file = TypeAdapter(SimcoreS3FileID).validate_python( + f"{faker.uuid4()}/{faker.uuid4()}/project_id_is_missing" ) assert ( await storage_s3_client.object_exists( @@ -1110,7 +1125,7 @@ async def test_download_file_access_rights( @pytest.mark.parametrize( "file_size", [ - pytest.param(parse_obj_as(ByteSize, "1Mib")), + pytest.param(TypeAdapter(ByteSize).validate_python("1Mib")), ], ids=byte_size_ids, ) @@ -1184,7 +1199,7 @@ async def test_copy_as_soft_link( # now let's try with whatever link id file, original_file_uuid = await upload_file( - parse_obj_as(ByteSize, "10Mib"), faker.file_name() + TypeAdapter(ByteSize).validate_python("10Mib"), faker.file_name() ) url = ( client.app.router["copy_as_soft_link"] @@ -1193,13 +1208,15 @@ async def test_copy_as_soft_link( ) .with_query(user_id=user_id) ) - link_id = SimcoreS3FileID(f"api/{node_id}/{faker.file_name()}") + link_id = TypeAdapter(SimcoreS3FileID).validate_python( + f"api/{node_id}/{faker.file_name()}" + ) response = await client.post( f"{url}", json=jsonable_encoder(SoftCopyBody(link_id=link_id)) ) data, error = await assert_status(response, status.HTTP_200_OK) assert not error - fmd = parse_obj_as(FileMetaDataGet, data) + fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert fmd.file_id == link_id @@ -1223,7 +1240,7 @@ async def __list_files( response = await client.get(f"{get_url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - return parse_obj_as(list[FileMetaDataGet], data) + return TypeAdapter(list[FileMetaDataGet]).validate_python(data) async def _list_files_legacy( @@ -1257,9 +1274,9 @@ async def _list_files_and_directories( @pytest.mark.parametrize( "file_size", [ - parse_obj_as(ByteSize, "-1"), - parse_obj_as(ByteSize, "0"), - parse_obj_as(ByteSize, "1TB"), + ByteSize(-1), + TypeAdapter(ByteSize).validate_python("0"), + TypeAdapter(ByteSize).validate_python("1TB"), ], ) async def test_is_directory_link_forces_link_type_and_size( @@ -1329,7 +1346,7 @@ async def test_upload_file_is_directory_and_remove_content( location_id: LocationID, user_id: UserID, ): - FILE_SIZE_IN_DIR = parse_obj_as(ByteSize, "1Mib") + FILE_SIZE_IN_DIR = TypeAdapter(ByteSize).validate_python("1Mib") DIR_NAME = "some-dir" SUBDIR_COUNT = 4 FILE_COUNT = 5 @@ -1396,7 +1413,7 @@ async def test_listing_more_than_1000_objects_in_bucket( ): async with create_directory_with_files( dir_name="some-random", - file_size_in_dir=parse_obj_as(ByteSize, "1"), + file_size_in_dir=TypeAdapter(ByteSize).validate_python("1"), subdir_count=1, file_count=files_in_dir, ) as directory_file_upload: @@ -1427,19 +1444,19 @@ async def test_listing_with_project_id_filter( project, src_projects_list = await random_project_with_files( num_nodes=1, file_sizes=(ByteSize(1),), - file_checksums=(SHA256Str(faker.sha256()),), + file_checksums=(TypeAdapter(SHA256Str).validate_python(faker.sha256()),), ) _, _ = await random_project_with_files( num_nodes=1, file_sizes=(ByteSize(1),), - file_checksums=(SHA256Str(faker.sha256()),), + file_checksums=(TypeAdapter(SHA256Str).validate_python(faker.sha256()),), ) assert len(src_projects_list.keys()) > 0 - node_id = list(src_projects_list.keys())[0] + node_id = next(iter(src_projects_list.keys())) project_files_in_db = set(src_projects_list[node_id]) assert len(project_files_in_db) > 0 project_id = project["uuid"] - project_file_name = Path(choice(list(project_files_in_db))).name + project_file_name = Path(choice(list(project_files_in_db))).name # noqa: S311 assert client.app query = { @@ -1456,7 +1473,7 @@ async def test_listing_with_project_id_filter( response = await client.get(f"{url}") data, _ = await assert_status(response, status.HTTP_200_OK) - list_of_files = parse_obj_as(list[FileMetaDataGet], data) + list_of_files = TypeAdapter(list[FileMetaDataGet]).validate_python(data) if uuid_filter: assert len(list_of_files) == 1 diff --git a/services/storage/tests/unit/test_handlers_files_metadata.py b/services/storage/tests/unit/test_handlers_files_metadata.py index dd0c8138ebb..9abd834d21a 100644 --- a/services/storage/tests/unit/test_handlers_files_metadata.py +++ b/services/storage/tests/unit/test_handlers_files_metadata.py @@ -16,7 +16,7 @@ from models_library.api_schemas_storage import FileMetaDataGet, SimcoreS3FileID from models_library.projects import ProjectID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from servicelib.aiohttp import status @@ -58,12 +58,12 @@ async def test_get_files_metadata( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds # now add some stuff there NUM_FILES = 10 - file_size = parse_obj_as(ByteSize, "15Mib") + file_size = TypeAdapter(ByteSize).validate_python("15Mib") files_owned_by_us = [ await upload_file(file_size, faker.file_name()) for _ in range(NUM_FILES) ] @@ -73,7 +73,7 @@ async def test_get_files_metadata( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == NUM_FILES # checks project_id filter! @@ -90,13 +90,13 @@ async def test_get_files_metadata( previous_data = deepcopy(data) data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (NUM_FILES) assert previous_data == data # create some more files but with a base common name NUM_FILES = 10 - file_size = parse_obj_as(ByteSize, "15Mib") + file_size = TypeAdapter(ByteSize).validate_python("15Mib") files_with_common_name = [ await upload_file(file_size, f"common_name-{faker.file_name()}") for _ in range(NUM_FILES) @@ -107,14 +107,14 @@ async def test_get_files_metadata( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (2 * NUM_FILES) # we can filter them now response = await client.get(f"{url.update_query(uuid_filter='common_name')}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == (NUM_FILES) @@ -171,7 +171,7 @@ async def test_get_file_metadata( # now add some stuff there NUM_FILES = 10 - file_size = parse_obj_as(ByteSize, "15Mib") + file_size = TypeAdapter(ByteSize).validate_python("15Mib") files_owned_by_us = [] for _ in range(NUM_FILES): files_owned_by_us.append(await upload_file(file_size, faker.file_name())) @@ -188,6 +188,6 @@ async def test_get_file_metadata( data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - fmd = parse_obj_as(FileMetaDataGet, data) + fmd = TypeAdapter(FileMetaDataGet).validate_python(data) assert fmd.file_id == selected_file_uuid assert fmd.file_size == selected_file.stat().st_size diff --git a/services/storage/tests/unit/test_handlers_health.py b/services/storage/tests/unit/test_handlers_health.py index d10b882b611..8705c4c8e36 100644 --- a/services/storage/tests/unit/test_handlers_health.py +++ b/services/storage/tests/unit/test_handlers_health.py @@ -26,7 +26,7 @@ async def test_health_check(client: TestClient): assert data assert not error - app_health = HealthCheck.parse_obj(data) + app_health = HealthCheck.model_validate(data) assert app_health.name == simcore_service_storage._meta.PROJECT_NAME # noqa: SLF001 assert app_health.version == str( simcore_service_storage._meta.VERSION @@ -41,7 +41,7 @@ async def test_health_status(client: TestClient): assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert ( app_status_check.app_name == simcore_service_storage._meta.PROJECT_NAME ) # noqa: SLF001 @@ -68,7 +68,7 @@ async def test_bad_health_status_if_bucket_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" # now delete the bucket await s3_client.delete_bucket(Bucket=storage_s3_bucket) @@ -77,7 +77,7 @@ async def test_bad_health_status_if_bucket_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "no access to S3 bucket" @@ -90,7 +90,7 @@ async def test_bad_health_status_if_s3_server_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" # now disable the s3 server mocked_aws_server.stop() @@ -99,7 +99,7 @@ async def test_bad_health_status_if_s3_server_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "failed" # start the server again mocked_aws_server.start() @@ -108,5 +108,5 @@ async def test_bad_health_status_if_s3_server_missing( data, error = await assert_status(response, status.HTTP_200_OK) assert data assert not error - app_status_check = AppStatusCheck.parse_obj(data) + app_status_check = AppStatusCheck.model_validate(data) assert app_status_check.services["s3"]["healthy"] == "connected" diff --git a/services/storage/tests/unit/test_handlers_simcore_s3.py b/services/storage/tests/unit/test_handlers_simcore_s3.py index e922f1f60f0..bcda9331f2b 100644 --- a/services/storage/tests/unit/test_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_handlers_simcore_s3.py @@ -26,7 +26,7 @@ from models_library.projects_nodes_io import NodeID, NodeIDStr, SimcoreS3FileID from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.logging_tools import log_context from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -82,7 +82,7 @@ async def test_simcore_s3_access_returns_default(client: TestClient): data, error = await assert_status(response, status.HTTP_200_OK) assert not error assert data - received_settings = S3Settings.parse_obj(data) + received_settings = S3Settings.model_validate(data) assert received_settings @@ -209,12 +209,12 @@ async def test_copy_folders_from_valid_project_with_one_large_file( ], ): # 1. create a src project with 1 large file - sha256_checksum: SHA256Str = parse_obj_as( - SHA256Str, "0b3216d95ec5a36c120ba16c88911dcf5ff655925d0fbdbc74cf95baf86de6fc" + sha256_checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python( + "0b3216d95ec5a36c120ba16c88911dcf5ff655925d0fbdbc74cf95baf86de6fc" ) src_project, src_projects_list = await random_project_with_files( 1, - (parse_obj_as(ByteSize, "210Mib"),), + (TypeAdapter(ByteSize).validate_python("210Mib"),), (sha256_checksum,), ) # 2. create a dst project without files @@ -233,7 +233,9 @@ async def test_copy_folders_from_valid_project_with_one_large_file( ) # check that file meta data was effectively copied for src_node_id in src_projects_list: - dst_node_id = nodes_map.get(NodeIDStr(f"{src_node_id}")) + dst_node_id = nodes_map.get( + TypeAdapter(NodeIDStr).validate_python(f"{src_node_id}") + ) assert dst_node_id for src_file_id, src_file in src_projects_list[src_node_id].items(): path: Any = src_file["path"] @@ -242,17 +244,18 @@ async def test_copy_folders_from_valid_project_with_one_large_file( assert isinstance(checksum, str) await assert_file_meta_data_in_db( aiopg_engine, - file_id=parse_obj_as( - SimcoreS3FileID, + file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{src_file_id}".replace( src_project["uuid"], dst_project["uuid"] - ).replace(f"{src_node_id}", f"{dst_node_id}"), + ).replace(f"{src_node_id}", f"{dst_node_id}") ), expected_entry_exists=True, expected_file_size=path.stat().st_size, expected_upload_id=None, expected_upload_expiration_date=None, - expected_sha256_checksum=SHA256Str(checksum), + expected_sha256_checksum=TypeAdapter(SHA256Str).validate_python( + checksum + ), ) @@ -292,7 +295,9 @@ async def test_copy_folders_from_valid_project( # check that file meta data was effectively copied for src_node_id in src_projects_list: - dst_node_id = nodes_map.get(NodeIDStr(f"{src_node_id}")) + dst_node_id = nodes_map.get( + TypeAdapter(NodeIDStr).validate_python(f"{src_node_id}") + ) assert dst_node_id for src_file_id, src_file in src_projects_list[src_node_id].items(): path: Any = src_file["path"] @@ -301,17 +306,18 @@ async def test_copy_folders_from_valid_project( assert isinstance(checksum, str) await assert_file_meta_data_in_db( aiopg_engine, - file_id=parse_obj_as( - SimcoreS3FileID, + file_id=TypeAdapter(SimcoreS3FileID).validate_python( f"{src_file_id}".replace( src_project["uuid"], dst_project["uuid"] - ).replace(f"{src_node_id}", f"{dst_node_id}"), + ).replace(f"{src_node_id}", f"{dst_node_id}") ), expected_entry_exists=True, expected_file_size=path.stat().st_size, expected_upload_id=None, expected_upload_expiration_date=None, - expected_sha256_checksum=SHA256Str(checksum), + expected_sha256_checksum=TypeAdapter(SHA256Str).validate_python( + checksum + ), ) @@ -394,9 +400,9 @@ async def with_random_project_with_files( ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],]: return await random_project_with_files( file_sizes=( - parse_obj_as(ByteSize, "1Mib"), - parse_obj_as(ByteSize, "2Mib"), - parse_obj_as(ByteSize, "5Mib"), + TypeAdapter(ByteSize).validate_python("1Mib"), + TypeAdapter(ByteSize).validate_python("2Mib"), + TypeAdapter(ByteSize).validate_python("5Mib"), ) ) @@ -472,7 +478,7 @@ async def uploaded_file_ids( for _ in range(expected_number_of_user_files): file_path, file_id = await upload_file( - file_size=parse_obj_as(ByteSize, "10Mib"), + file_size=TypeAdapter(ByteSize).validate_python("10Mib"), file_name=faker.file_name(), sha256_checksum=faker.sha256(), ) @@ -525,7 +531,7 @@ async def test_search_files_request( data, error = await assert_status(response, status.HTTP_200_OK) assert not error - found = parse_obj_as(list[FileMetaDataGet], data) + found = TypeAdapter(list[FileMetaDataGet]).validate_python(data) expected = uploaded_file_ids[ search_files_query_params.offset : search_files_query_params.offset @@ -548,7 +554,7 @@ async def test_search_files( ): assert client.app _file_name: str = faker.file_name() - _sha256_checksum: SHA256Str = parse_obj_as(SHA256Str, faker.sha256()) + _sha256_checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python(faker.sha256()) url = ( client.app.router["search_files"] @@ -571,12 +577,12 @@ async def test_search_files( data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds # let's upload some files now file, file_id = await upload_file( - file_size=parse_obj_as(ByteSize, "10Mib"), + file_size=TypeAdapter(ByteSize).validate_python("10Mib"), file_name=_file_name, sha256_checksum=_sha256_checksum, ) @@ -584,7 +590,7 @@ async def test_search_files( response = await client.post(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == 1 assert list_fmds[0].file_id == file_id assert list_fmds[0].file_size == file.stat().st_size @@ -600,7 +606,7 @@ async def test_search_files( response = await client.post(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert len(list_fmds) == 1 assert list_fmds[0].file_id == file_id assert list_fmds[0].file_size == file.stat().st_size @@ -620,5 +626,5 @@ async def test_search_files( response = await client.post(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) assert not error - list_fmds = parse_obj_as(list[FileMetaDataGet], data) + list_fmds = TypeAdapter(list[FileMetaDataGet]).validate_python(data) assert not list_fmds diff --git a/services/storage/tests/unit/test_models.py b/services/storage/tests/unit/test_models.py index 0dbab6821d2..250b037e5cf 100644 --- a/services/storage/tests/unit/test_models.py +++ b/services/storage/tests/unit/test_models.py @@ -43,7 +43,7 @@ def test_file_id(file_id: str): assert parsed_file_id == file_id -def test_fmd_build(): +def test_fmd_build_api(): file_id = TypeAdapter(SimcoreS3FileID).validate_python(f"api/{uuid.uuid4()}/xx.dat") fmd = FileMetaData.from_simcore_node( user_id=12, @@ -64,6 +64,8 @@ def test_fmd_build(): assert fmd.location_id == SimcoreS3DataManager.get_location_id() assert fmd.bucket_name == "test-bucket" + +def test_fmd_build_webapi(): file_id = TypeAdapter(SimcoreS3FileID).validate_python( f"{uuid.uuid4()}/{uuid.uuid4()}/xx.dat" ) diff --git a/services/storage/tests/unit/test_simcore_s3_dsm.py b/services/storage/tests/unit/test_simcore_s3_dsm.py index 3f360b1505a..41c69355025 100644 --- a/services/storage/tests/unit/test_simcore_s3_dsm.py +++ b/services/storage/tests/unit/test_simcore_s3_dsm.py @@ -12,7 +12,7 @@ from models_library.basic_types import SHA256Str from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from simcore_service_storage import db_file_meta_data from simcore_service_storage.models import FileMetaData from simcore_service_storage.s3 import get_s3_client @@ -24,7 +24,7 @@ @pytest.fixture def file_size() -> ByteSize: - return parse_obj_as(ByteSize, "1") + return TypeAdapter(ByteSize).validate_python("1") @pytest.fixture @@ -47,7 +47,9 @@ async def test__copy_path_s3_s3( aiopg_engine: Engine, ): def _get_dest_file_id(src: SimcoreS3FileID) -> SimcoreS3FileID: - return parse_obj_as(SimcoreS3FileID, f"{Path(src).parent}/the-copy") + return TypeAdapter(SimcoreS3FileID).validate_python( + f"{Path(src).parent}/the-copy" + ) async def _copy_s3_path(s3_file_id_to_copy: SimcoreS3FileID) -> None: async with aiopg_engine.acquire() as conn: @@ -84,7 +86,7 @@ async def _count_files(s3_file_id: SimcoreS3FileID, expected_count: int) -> None assert directory_file_upload.urls[0].path s3_object = directory_file_upload.urls[0].path.lstrip("/") - s3_file_id_dir_src = parse_obj_as(SimcoreS3FileID, s3_object) + s3_file_id_dir_src = TypeAdapter(SimcoreS3FileID).validate_python(s3_object) s3_file_id_dir_dst = _get_dest_file_id(s3_file_id_dir_src) await _count_files(s3_file_id_dir_dst, expected_count=0) @@ -104,7 +106,7 @@ async def test_upload_and_search( user_id: UserID, faker: Faker, ): - checksum: SHA256Str = parse_obj_as(SHA256Str, faker.sha256()) + checksum: SHA256Str = TypeAdapter(SHA256Str).validate_python(faker.sha256()) _, _ = await upload_file(file_size, "file1", sha256_checksum=checksum) _, _ = await upload_file(file_size, "file2", sha256_checksum=checksum) diff --git a/services/storage/tests/unit/test_utils.py b/services/storage/tests/unit/test_utils.py index 90dca22d42d..d0aef454df1 100644 --- a/services/storage/tests/unit/test_utils.py +++ b/services/storage/tests/unit/test_utils.py @@ -14,9 +14,10 @@ import pytest from aiohttp import ClientSession from faker import Faker +from models_library.api_schemas_storage import UNDEFINED_SIZE_TYPE from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, SimcoreS3FileID -from pydantic import ByteSize, HttpUrl, TypeAdapter, parse_obj_as +from pydantic import ByteSize, HttpUrl, TypeAdapter from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID from simcore_service_storage.models import ETag, FileMetaData, S3BucketName, UploadID from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager @@ -46,24 +47,30 @@ async def test_download_files(tmp_path: Path, httpbin_base_url: HttpUrl): [ (-1, None, None, None, False), (0, None, None, None, False), - (random.randint(1, 1000000), None, None, None, False), + (random.randint(1, 1000000), None, None, None, False), # noqa: S311 (-1, "some_valid_entity_tag", None, None, False), (0, "some_valid_entity_tag", None, None, False), ( - random.randint(1, 1000000), + random.randint(1, 1000000), # noqa: S311 "some_valid_entity_tag", "som_upload_id", None, False, ), ( - random.randint(1, 1000000), + random.randint(1, 1000000), # noqa: S311 "some_valid_entity_tag", None, - datetime.datetime.utcnow(), + datetime.datetime.now(datetime.UTC), False, ), - (random.randint(1, 1000000), "some_valid_entity_tag", None, None, True), + ( + random.randint(1, 1000000), # noqa: S311 + "some_valid_entity_tag", + None, + None, + True, + ), ], ) def test_file_entry_valid( @@ -84,7 +91,9 @@ def test_file_entry_valid( location_name=SimcoreS3DataManager.get_location_name(), sha256_checksum=None, ) - fmd.file_size = parse_obj_as(ByteSize, file_size) + fmd.file_size = TypeAdapter(UNDEFINED_SIZE_TYPE | ByteSize).validate_python( + file_size + ) fmd.entity_tag = entity_tag fmd.upload_id = upload_id fmd.upload_expires_at = upload_expires_at diff --git a/services/storage/tests/unit/test_utils_handlers.py b/services/storage/tests/unit/test_utils_handlers.py index a5f82a6b893..cc220ceb3e2 100644 --- a/services/storage/tests/unit/test_utils_handlers.py +++ b/services/storage/tests/unit/test_utils_handlers.py @@ -31,8 +31,7 @@ async def raising_handler( @pytest.fixture def mock_request(mocker: MockerFixture) -> web.Request: - mock = mocker.patch("aiohttp.web.Request", autospec=True) - return mock + return mocker.patch("aiohttp.web.Request", autospec=True) class FakeErrorModel(BaseModel): @@ -48,7 +47,10 @@ class FakeErrorModel(BaseModel): (ProjectNotFoundError(project_id="x"), web.HTTPNotFound), (FileAccessRightError(file_id="x", access_right="x"), web.HTTPForbidden), (ProjectAccessRightError(project_id="x", access_right="x"), web.HTTPForbidden), - (ValidationError(errors=[], model=FakeErrorModel), web.HTTPUnprocessableEntity), + ( + ValidationError.from_exception_data(title="test", line_errors=[]), + web.HTTPUnprocessableEntity, + ), (DBAPIError, web.HTTPServiceUnavailable), ], )